{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First ensemble with NSL-KDD\n",
    "# Parameters\n",
    "# Few parameters are not fully implemented yet\n",
    "\n",
    "#----------------------------------------------\n",
    "# 0 for not using it as base learner\n",
    "# 1 for using it as base learner\n",
    "# not implemented but in the code in someparts\n",
    "use_model_ada = 1 \n",
    "use_model_dnn = 1 \n",
    "use_model_mlp = 1 \n",
    "use_model_lgbm = 1 \n",
    "use_model_rf = 1 \n",
    "use_model_svm = 1\n",
    "use_model_knn = 1 \n",
    "#----------------------------------------------\n",
    "# 0 for training the model\n",
    "# 1 for using the saved version of the model\n",
    "\n",
    "# load_model_ada = 0 \n",
    "# load_model_dnn = 0 \n",
    "# load_model_mlp = 0 \n",
    "# load_model_lgbm = 0 \n",
    "# load_model_rf = 0 \n",
    "# load_model_svm = 0\n",
    "# load_model_knn = 0 \n",
    "#----------------------------------------------\n",
    "# not implemented but in the code in someparts\n",
    "load_model_ada = 1\n",
    "load_model_dnn = 1 \n",
    "load_model_mlp = 1 \n",
    "load_model_lgbm = 1 \n",
    "load_model_rf = 1                               \n",
    "load_model_svm = 1\n",
    "load_model_knn = 1 \n",
    "#----------------------------------------------\n",
    "\n",
    "# Implemented\n",
    "#----------------------------------------------\n",
    "# feature_selection_bit = 0 # OFF\n",
    "feature_selection_bit = 1 # On\n",
    "pick_prob = 1 # set equal one to choose the dataset with probabilities, set to 0 to choose one with the classes.\n",
    "# pick_prob = 0\n",
    "generate_feature_importance = 0 # Generate Shap graphs\n",
    "\n",
    "\n",
    "column_features = [\n",
    "                    'dnn',\n",
    "                   'rf',\n",
    "                   'lgbm',\n",
    "                   'ada',\n",
    "                   'knn',\n",
    "                   'mlp',\n",
    "                   'svm',\n",
    "                   'cat',\n",
    "                   'xgb',\n",
    "                   'lr',\n",
    "                   'dt',\n",
    "                   'label']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specify the name of the output text file\n",
    "if feature_selection_bit == 0:\n",
    "\n",
    "    if pick_prob == 0:\n",
    "        output_file_name = \"ensemble_level_01_all_features_classes_dt.txt\"\n",
    "        with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "        with open(output_file_name, \"a\") as f: print('----ensemble_level_01_all_features_classes--', file = f)\n",
    "\n",
    "    elif pick_prob == 1:\n",
    "        output_file_name = \"ensemble_level_01_all_features_probabilites_dt.txt\"\n",
    "        with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "        with open(output_file_name, \"a\") as f: print('----ensemble_level_01_all_features_probabilites--', file = f)\n",
    "\n",
    "elif feature_selection_bit == 1:\n",
    "    if pick_prob == 0:\n",
    "        output_file_name = \"ensemble_level_01_feature_selection_classes_dt.txt\"\n",
    "        with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "        with open(output_file_name, \"a\") as f: print('----ensemble_level_01_feature_selection_classes--', file = f)\n",
    "    elif pick_prob == 1:\n",
    "        output_file_name = \"ensemble_level_01_feature_selection_probabilites_dt.txt\"\n",
    "        with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "        with open(output_file_name, \"a\") as f: print('----ensemble_level_01_feature_selection_probabilites--', file = f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/env python\n",
    "# coding: utf-8\n",
    "\n",
    "# In[1]:\n",
    "# importing required libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle # saving and loading trained model\n",
    "from os import path\n",
    "\n",
    "\n",
    "# importing required libraries for normalizing data\n",
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import (StandardScaler, OrdinalEncoder,LabelEncoder, MinMaxScaler, OneHotEncoder)\n",
    "from sklearn.preprocessing import Normalizer, MaxAbsScaler , RobustScaler, PowerTransformer\n",
    "\n",
    "# importing library for plotting\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import accuracy_score # for calculating accuracy of model\n",
    "from sklearn.model_selection import train_test_split # for splitting the dataset for training and testing\n",
    "from sklearn.metrics import classification_report # for generating a classification report of model\n",
    "\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "\n",
    "from keras.layers import Dense # importing dense layer\n",
    "\n",
    "from keras.layers import Input\n",
    "from keras.models import Model\n",
    "# representation of model layers\n",
    "#from keras.utils import plot_model\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import shap\n",
    "import time\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import balanced_accuracy_score\n",
    "from sklearn.metrics import matthews_corrcoef\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "import joblib\n",
    "from sklearn.model_selection import train_test_split\n",
    "import sklearn\n",
    "from tabulate import tabulate\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/env python\n",
    "# coding: utf-8\n",
    "\n",
    "# In[1]:\n",
    "# importing required libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle # saving and loading trained model\n",
    "from os import path\n",
    "\n",
    "\n",
    "# importing required libraries for normalizing data\n",
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import (StandardScaler, OrdinalEncoder,LabelEncoder, MinMaxScaler, OneHotEncoder)\n",
    "from sklearn.preprocessing import Normalizer, MaxAbsScaler , RobustScaler, PowerTransformer\n",
    "\n",
    "# importing library for plotting\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import accuracy_score # for calculating accuracy of model\n",
    "from sklearn.model_selection import train_test_split # for splitting the dataset for training and testing\n",
    "from sklearn.metrics import classification_report # for generating a classification report of model\n",
    "\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "\n",
    "from keras.layers import Dense # importing dense layer\n",
    "\n",
    "from keras.layers import Input\n",
    "from keras.models import Model\n",
    "# representation of model layers\n",
    "#from keras.utils import plot_model\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import shap\n",
    "\n",
    "import time\n",
    "start_program = time.time()\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import balanced_accuracy_score\n",
    "from sklearn.metrics import matthews_corrcoef\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "import joblib\n",
    "from sklearn.model_selection import train_test_split\n",
    "import sklearn\n",
    "from tabulate import tabulate\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def confusion_metrics (name_model,predictions,true_labels,time_taken):\n",
    "\n",
    "    name = name_model\n",
    "    pred_label = predictions\n",
    "    y_test_01 = true_labels \n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(name, file = f)\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('CONFUSION MATRIX')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    # pred_label = label[ypred]\n",
    "\n",
    "    confusion_matrix = pd.crosstab(y_test_01, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "    all_unique_values = sorted(set(pred_label) | set(y_test_01))\n",
    "    z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "    rows, cols = confusion_matrix.shape\n",
    "    z[:rows, :cols] = confusion_matrix\n",
    "    confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "    # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "    # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "    print(confusion_matrix)\n",
    "    with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "    FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "    FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "    TP = np.diag(confusion_matrix)\n",
    "    TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "    TP_total = sum(TP)\n",
    "    TN_total = sum(TN)\n",
    "    FP_total = sum(FP)\n",
    "    FN_total = sum(FN)\n",
    "\n",
    "    TP_total = np.array(TP_total,dtype=np.float64)\n",
    "    TN_total = np.array(TN_total,dtype=np.float64)\n",
    "    FP_total = np.array(FP_total,dtype=np.float64)\n",
    "    FN_total = np.array(FN_total,dtype=np.float64)\n",
    "    FPR = FP / (FP + TN)\n",
    "    FPR = 100*(sum(FPR)/len(FPR))\n",
    "\n",
    "    #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('METRICS')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    Acc = accuracy_score(y_test_01, pred_label)\n",
    "    Precision = precision_score(y_test_01, pred_label, average='macro')\n",
    "    Recall = recall_score(y_test_01, pred_label, average='macro')\n",
    "    F1 =  f1_score(y_test_01, pred_label, average='macro')\n",
    "    BACC = balanced_accuracy_score(y_test_01, pred_label)\n",
    "    MCC = matthews_corrcoef(y_test_01, pred_label)\n",
    "\n",
    "\n",
    "    # voting_acc_01 = Acc\n",
    "    # voting_pre_01 = Precision\n",
    "    # weighed_avg_rec_01 = Recall\n",
    "    # weighed_avg_f1_01 = F1\n",
    "    # weighed_avg_bacc_01 = BACC\n",
    "    # weighed_avg_mcc_01 = MCC\n",
    "    # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "    print('Accuracy total: ', Acc)\n",
    "    print('Precision total: ', Precision )\n",
    "    print('Recall total: ', Recall )\n",
    "    print('F1 total: ', F1 )\n",
    "    print('BACC total: ', BACC)\n",
    "    print('MCC total: ', MCC)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Time Taken: ', time_taken, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('FPR: ', FPR, '%' ,file = f)\n",
    "\n",
    "    return Acc, Precision, Recall, F1, BACC, MCC, FPR\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "df_level_00_1=pd.read_csv('base_models_prob_feature_selection.csv')\n",
    "df_level_00_0=pd.read_csv('base_models_class_feature_selection.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dnn</th>\n",
       "      <th>rf</th>\n",
       "      <th>lgbm</th>\n",
       "      <th>ada</th>\n",
       "      <th>knn</th>\n",
       "      <th>mlp</th>\n",
       "      <th>svm</th>\n",
       "      <th>cat</th>\n",
       "      <th>xgb</th>\n",
       "      <th>lr</th>\n",
       "      <th>dt</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.182097</td>\n",
       "      <td>0.449534</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.170281</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.991302</td>\n",
       "      <td>0.091089</td>\n",
       "      <td>0.984845</td>\n",
       "      <td>0.987333</td>\n",
       "      <td>0.279080</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.821370</td>\n",
       "      <td>0.981712</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.177443</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.377378</td>\n",
       "      <td>0.998089</td>\n",
       "      <td>0.996417</td>\n",
       "      <td>0.998907</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.809624</td>\n",
       "      <td>0.929960</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.177443</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.270555</td>\n",
       "      <td>0.999135</td>\n",
       "      <td>0.991678</td>\n",
       "      <td>0.999997</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.822028</td>\n",
       "      <td>0.935771</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.177443</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999884</td>\n",
       "      <td>0.264639</td>\n",
       "      <td>0.997208</td>\n",
       "      <td>0.981911</td>\n",
       "      <td>0.949104</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.727856</td>\n",
       "      <td>0.562347</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.177443</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.997751</td>\n",
       "      <td>0.263482</td>\n",
       "      <td>0.899433</td>\n",
       "      <td>0.894973</td>\n",
       "      <td>0.202297</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82974</th>\n",
       "      <td>0.794754</td>\n",
       "      <td>0.961144</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.177443</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999998</td>\n",
       "      <td>0.263681</td>\n",
       "      <td>0.997118</td>\n",
       "      <td>0.997761</td>\n",
       "      <td>0.988671</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82975</th>\n",
       "      <td>0.812500</td>\n",
       "      <td>0.981712</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.177443</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.448115</td>\n",
       "      <td>0.997962</td>\n",
       "      <td>0.996417</td>\n",
       "      <td>0.997908</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82976</th>\n",
       "      <td>0.835878</td>\n",
       "      <td>0.981712</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.177443</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999997</td>\n",
       "      <td>0.572470</td>\n",
       "      <td>0.998085</td>\n",
       "      <td>0.996417</td>\n",
       "      <td>0.999512</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82977</th>\n",
       "      <td>0.898158</td>\n",
       "      <td>0.996759</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.221467</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.427222</td>\n",
       "      <td>0.996935</td>\n",
       "      <td>0.995946</td>\n",
       "      <td>0.999805</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82978</th>\n",
       "      <td>0.818426</td>\n",
       "      <td>0.981712</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.177443</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999998</td>\n",
       "      <td>0.264610</td>\n",
       "      <td>0.999040</td>\n",
       "      <td>0.997243</td>\n",
       "      <td>0.998411</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>82979 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            dnn        rf  lgbm       ada  knn       mlp       svm       cat  \\\n",
       "0      0.182097  0.449534   1.0  0.170281  1.0  0.991302  0.091089  0.984845   \n",
       "1      0.821370  0.981712   1.0  0.177443  1.0  0.999999  0.377378  0.998089   \n",
       "2      0.809624  0.929960   1.0  0.177443  1.0  1.000000  0.270555  0.999135   \n",
       "3      0.822028  0.935771   1.0  0.177443  1.0  0.999884  0.264639  0.997208   \n",
       "4      0.727856  0.562347   1.0  0.177443  1.0  0.997751  0.263482  0.899433   \n",
       "...         ...       ...   ...       ...  ...       ...       ...       ...   \n",
       "82974  0.794754  0.961144   1.0  0.177443  1.0  0.999998  0.263681  0.997118   \n",
       "82975  0.812500  0.981712   1.0  0.177443  1.0  1.000000  0.448115  0.997962   \n",
       "82976  0.835878  0.981712   1.0  0.177443  1.0  0.999997  0.572470  0.998085   \n",
       "82977  0.898158  0.996759   1.0  0.221467  1.0  1.000000  0.427222  0.996935   \n",
       "82978  0.818426  0.981712   1.0  0.177443  1.0  0.999998  0.264610  0.999040   \n",
       "\n",
       "            xgb        lr   dt  label  \n",
       "0      0.987333  0.279080  1.0    1.0  \n",
       "1      0.996417  0.998907  1.0    6.0  \n",
       "2      0.991678  0.999997  1.0    6.0  \n",
       "3      0.981911  0.949104  1.0    6.0  \n",
       "4      0.894973  0.202297  1.0    6.0  \n",
       "...         ...       ...  ...    ...  \n",
       "82974  0.997761  0.988671  1.0    6.0  \n",
       "82975  0.996417  0.997908  1.0    6.0  \n",
       "82976  0.996417  0.999512  1.0    6.0  \n",
       "82977  0.995946  0.999805  1.0    1.0  \n",
       "82978  0.997243  0.998411  1.0    6.0  \n",
       "\n",
       "[82979 rows x 12 columns]"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_level_00_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "y1 = df_level_00_1.pop('label')\n",
    "X1 = df_level_00_1\n",
    "df_level_00_1 = X1.assign(label = y1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "y0 = df_level_00_0.pop('label')\n",
    "X0 = df_level_00_0\n",
    "df_level_00_0 = X0.assign(label = y0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [],
   "source": [
    "if feature_selection_bit == 1:\n",
    "\n",
    "    from sklearn.feature_selection import mutual_info_classif\n",
    "    %matplotlib inline\n",
    "\n",
    "    # Compute information gain using mutual information\n",
    "    importances0 = mutual_info_classif(X0, y0)\n",
    "    importances1 = mutual_info_classif(X1, y1)\n",
    "\n",
    "\n",
    "    feat_importances0 = pd.Series(importances0, df_level_00_0.columns[0:len(df_level_00_0.columns)-1])\n",
    "    feat_importances1= pd.Series(importances1, df_level_00_1.columns[0:len(df_level_00_1.columns)-1])\n",
    "\n",
    "    # feat_importances.plot(kind='barh', color = 'teal')\n",
    "        \n",
    "    feat_importances_sorted0 = feat_importances0.sort_values( ascending=False)\n",
    "    feat_importances_sorted1 = feat_importances1.sort_values( ascending=False)\n",
    "\n",
    "\n",
    "    # Print or use the sorted DataFrame\n",
    "    print(feat_importances_sorted0)\n",
    "    print(feat_importances_sorted1)\n",
    "\n",
    "    # feat_importances_sorted.plot(kind='barh', color = 'teal')\n",
    "    # feat_importances_sorted\n",
    "    top_features0 = feat_importances_sorted0.nlargest(5)\n",
    "    top_features1 = feat_importances_sorted1.nlargest(5)\n",
    "\n",
    "    top_feature_names0 = top_features0.index.tolist()\n",
    "    top_feature_names1 = top_features1.index.tolist()\n",
    "\n",
    "\n",
    "    print(\"Top 5 feature names:\")\n",
    "    print(top_feature_names0)\n",
    "    print(top_feature_names1)\n",
    "\n",
    "    column_features0 = top_feature_names0\n",
    "    column_features1 = top_feature_names1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Assuming df is your DataFrame\n",
    "# if feature_selection_bit == 1:\n",
    "#     df_level_00_1=pd.read_csv('base_models_prob_feature_selection.csv',names=column_features)\n",
    "#     df_level_00_0=pd.read_csv('base_models_class_feature_selection.csv',names=column_features)\n",
    "\n",
    "# if feature_selection_bit == 0:\n",
    "\n",
    "#     df_level_00_1=pd.read_csv('base_models_prob_feature_selection.csv')\n",
    "#     df_level_00_0=pd.read_csv('base_models_class_feature_selection.csv')\n",
    "\n",
    "# df_level_00_1=pd.read_csv('base_models_prob_feature_selection.csv')\n",
    "# df_level_00_0=pd.read_csv('base_models_class_feature_selection.csv')\n",
    "\n",
    "if feature_selection_bit == 1:\n",
    "    df_level_00_0 = df_level_00_0[column_features0]\n",
    "    df_level_00_1 = df_level_00_1[column_features1]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dnn</th>\n",
       "      <th>rf</th>\n",
       "      <th>lgbm</th>\n",
       "      <th>ada</th>\n",
       "      <th>knn</th>\n",
       "      <th>mlp</th>\n",
       "      <th>svm</th>\n",
       "      <th>cat</th>\n",
       "      <th>xgb</th>\n",
       "      <th>lr</th>\n",
       "      <th>dt</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.182097</td>\n",
       "      <td>0.449534</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.170281</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.991302</td>\n",
       "      <td>0.091089</td>\n",
       "      <td>0.984845</td>\n",
       "      <td>0.987333</td>\n",
       "      <td>0.279080</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.821370</td>\n",
       "      <td>0.981712</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.177443</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.377378</td>\n",
       "      <td>0.998089</td>\n",
       "      <td>0.996417</td>\n",
       "      <td>0.998907</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.809624</td>\n",
       "      <td>0.929960</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.177443</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.270555</td>\n",
       "      <td>0.999135</td>\n",
       "      <td>0.991678</td>\n",
       "      <td>0.999997</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.822028</td>\n",
       "      <td>0.935771</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.177443</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999884</td>\n",
       "      <td>0.264639</td>\n",
       "      <td>0.997208</td>\n",
       "      <td>0.981911</td>\n",
       "      <td>0.949104</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.727856</td>\n",
       "      <td>0.562347</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.177443</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.997751</td>\n",
       "      <td>0.263482</td>\n",
       "      <td>0.899433</td>\n",
       "      <td>0.894973</td>\n",
       "      <td>0.202297</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82974</th>\n",
       "      <td>0.794754</td>\n",
       "      <td>0.961144</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.177443</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999998</td>\n",
       "      <td>0.263681</td>\n",
       "      <td>0.997118</td>\n",
       "      <td>0.997761</td>\n",
       "      <td>0.988671</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82975</th>\n",
       "      <td>0.812500</td>\n",
       "      <td>0.981712</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.177443</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.448115</td>\n",
       "      <td>0.997962</td>\n",
       "      <td>0.996417</td>\n",
       "      <td>0.997908</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82976</th>\n",
       "      <td>0.835878</td>\n",
       "      <td>0.981712</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.177443</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999997</td>\n",
       "      <td>0.572470</td>\n",
       "      <td>0.998085</td>\n",
       "      <td>0.996417</td>\n",
       "      <td>0.999512</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82977</th>\n",
       "      <td>0.898158</td>\n",
       "      <td>0.996759</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.221467</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.427222</td>\n",
       "      <td>0.996935</td>\n",
       "      <td>0.995946</td>\n",
       "      <td>0.999805</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82978</th>\n",
       "      <td>0.818426</td>\n",
       "      <td>0.981712</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.177443</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999998</td>\n",
       "      <td>0.264610</td>\n",
       "      <td>0.999040</td>\n",
       "      <td>0.997243</td>\n",
       "      <td>0.998411</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>82979 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            dnn        rf  lgbm       ada  knn       mlp       svm       cat  \\\n",
       "0      0.182097  0.449534   1.0  0.170281  1.0  0.991302  0.091089  0.984845   \n",
       "1      0.821370  0.981712   1.0  0.177443  1.0  0.999999  0.377378  0.998089   \n",
       "2      0.809624  0.929960   1.0  0.177443  1.0  1.000000  0.270555  0.999135   \n",
       "3      0.822028  0.935771   1.0  0.177443  1.0  0.999884  0.264639  0.997208   \n",
       "4      0.727856  0.562347   1.0  0.177443  1.0  0.997751  0.263482  0.899433   \n",
       "...         ...       ...   ...       ...  ...       ...       ...       ...   \n",
       "82974  0.794754  0.961144   1.0  0.177443  1.0  0.999998  0.263681  0.997118   \n",
       "82975  0.812500  0.981712   1.0  0.177443  1.0  1.000000  0.448115  0.997962   \n",
       "82976  0.835878  0.981712   1.0  0.177443  1.0  0.999997  0.572470  0.998085   \n",
       "82977  0.898158  0.996759   1.0  0.221467  1.0  1.000000  0.427222  0.996935   \n",
       "82978  0.818426  0.981712   1.0  0.177443  1.0  0.999998  0.264610  0.999040   \n",
       "\n",
       "            xgb        lr   dt  label  \n",
       "0      0.987333  0.279080  1.0    1.0  \n",
       "1      0.996417  0.998907  1.0    6.0  \n",
       "2      0.991678  0.999997  1.0    6.0  \n",
       "3      0.981911  0.949104  1.0    6.0  \n",
       "4      0.894973  0.202297  1.0    6.0  \n",
       "...         ...       ...  ...    ...  \n",
       "82974  0.997761  0.988671  1.0    6.0  \n",
       "82975  0.996417  0.997908  1.0    6.0  \n",
       "82976  0.996417  0.999512  1.0    6.0  \n",
       "82977  0.995946  0.999805  1.0    1.0  \n",
       "82978  0.997243  0.998411  1.0    6.0  \n",
       "\n",
       "[82979 rows x 12 columns]"
      ]
     },
     "execution_count": 193,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_level_00_1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dnn</th>\n",
       "      <th>rf</th>\n",
       "      <th>lgbm</th>\n",
       "      <th>ada</th>\n",
       "      <th>knn</th>\n",
       "      <th>mlp</th>\n",
       "      <th>svm</th>\n",
       "      <th>cat</th>\n",
       "      <th>xgb</th>\n",
       "      <th>lr</th>\n",
       "      <th>dt</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82974</th>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82975</th>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82976</th>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82977</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82978</th>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>82979 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       dnn   rf  lgbm  ada  knn  mlp  svm  cat  xgb   lr   dt  label\n",
       "0      6.0  6.0   1.0  6.0  1.0  1.0  6.0  1.0  1.0  6.0  1.0    1.0\n",
       "1      6.0  6.0   6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0    6.0\n",
       "2      6.0  6.0   6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0    6.0\n",
       "3      6.0  6.0   6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0    6.0\n",
       "4      6.0  6.0   6.0  6.0  6.0  6.0  0.0  6.0  6.0  0.0  6.0    6.0\n",
       "...    ...  ...   ...  ...  ...  ...  ...  ...  ...  ...  ...    ...\n",
       "82974  6.0  6.0   6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0    6.0\n",
       "82975  6.0  6.0   6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0    6.0\n",
       "82976  6.0  6.0   6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0    6.0\n",
       "82977  1.0  1.0   1.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0    1.0\n",
       "82978  6.0  6.0   6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0    6.0\n",
       "\n",
       "[82979 rows x 12 columns]"
      ]
     },
     "execution_count": 194,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_level_00_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_level_01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [],
   "source": [
    "if pick_prob == 1:\n",
    "    df_level_01 = df_level_00_1\n",
    "else: \n",
    "    df_level_01 = df_level_00_0\n",
    "\n",
    "df_level_01 = df_level_01.assign(label = y1)\n",
    "\n",
    "y_01 = df_level_01.pop('label') \n",
    "    \n",
    "X_01 = df_level_01\n",
    "df_level_01 = df_level_01.assign(label = y_01)\n",
    "\n",
    "\n",
    "split = 0.7\n",
    "X_train_01,X_test_01, y_train_01, y_test_01 = sklearn.model_selection.train_test_split(X_01, y_01, train_size=split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# df_level_02 = pd.read_csv('base_models_class_feature_selection.csv')\n",
    "\n",
    "# df_level_02\n",
    "\n",
    "# y_02 = df_level_02.pop('label')\n",
    "# X_02 = df_level_02\n",
    "# df_level_02 = df_level_02.assign(label = y_01)\n",
    "\n",
    "\n",
    "# split = 0.7\n",
    "# X_train_02,X_test_02, y_train_02, y_test_02 = sklearn.model_selection.train_test_split(X_02, y_02, train_size=split)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training the stronger model - STACK level 01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [],
   "source": [
    "#----------------------------------------------------------------\n",
    "with open(output_file_name, \"a\") as f: print('Stack model - Strong learner - level 01', file = f)\n",
    "with open(output_file_name, \"a\") as f: print('-------------------------------------------------------', file = f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dnn</th>\n",
       "      <th>rf</th>\n",
       "      <th>lgbm</th>\n",
       "      <th>ada</th>\n",
       "      <th>knn</th>\n",
       "      <th>mlp</th>\n",
       "      <th>svm</th>\n",
       "      <th>cat</th>\n",
       "      <th>xgb</th>\n",
       "      <th>lr</th>\n",
       "      <th>dt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>82131</th>\n",
       "      <td>0.993089</td>\n",
       "      <td>0.999665</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.221467</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.677671</td>\n",
       "      <td>0.999100</td>\n",
       "      <td>0.997781</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69866</th>\n",
       "      <td>0.970626</td>\n",
       "      <td>0.999665</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.221467</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.379590</td>\n",
       "      <td>0.999389</td>\n",
       "      <td>0.996623</td>\n",
       "      <td>0.999471</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20341</th>\n",
       "      <td>0.839683</td>\n",
       "      <td>0.981712</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.177443</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999997</td>\n",
       "      <td>0.428971</td>\n",
       "      <td>0.998438</td>\n",
       "      <td>0.996581</td>\n",
       "      <td>0.999967</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79165</th>\n",
       "      <td>0.821524</td>\n",
       "      <td>0.981712</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.177443</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999996</td>\n",
       "      <td>0.558171</td>\n",
       "      <td>0.998496</td>\n",
       "      <td>0.996417</td>\n",
       "      <td>0.983103</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20549</th>\n",
       "      <td>0.826017</td>\n",
       "      <td>0.981712</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.177443</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.329137</td>\n",
       "      <td>0.998824</td>\n",
       "      <td>0.996581</td>\n",
       "      <td>0.999908</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39846</th>\n",
       "      <td>0.821232</td>\n",
       "      <td>0.927554</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.177443</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.996391</td>\n",
       "      <td>0.342791</td>\n",
       "      <td>0.998106</td>\n",
       "      <td>0.972167</td>\n",
       "      <td>0.950086</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45178</th>\n",
       "      <td>0.740471</td>\n",
       "      <td>0.808703</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.334974</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.930795</td>\n",
       "      <td>0.826558</td>\n",
       "      <td>0.888832</td>\n",
       "      <td>0.899891</td>\n",
       "      <td>0.754910</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25910</th>\n",
       "      <td>0.999317</td>\n",
       "      <td>0.999665</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.221467</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.672632</td>\n",
       "      <td>0.999148</td>\n",
       "      <td>0.997881</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4089</th>\n",
       "      <td>0.826035</td>\n",
       "      <td>0.981712</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.177443</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999995</td>\n",
       "      <td>0.308658</td>\n",
       "      <td>0.999101</td>\n",
       "      <td>0.997243</td>\n",
       "      <td>0.998960</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69547</th>\n",
       "      <td>0.997685</td>\n",
       "      <td>0.999665</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.221467</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.598654</td>\n",
       "      <td>0.999439</td>\n",
       "      <td>0.994601</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24894 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            dnn        rf  lgbm       ada  knn       mlp       svm       cat  \\\n",
       "82131  0.993089  0.999665   1.0  0.221467  1.0  1.000000  0.677671  0.999100   \n",
       "69866  0.970626  0.999665   1.0  0.221467  1.0  1.000000  0.379590  0.999389   \n",
       "20341  0.839683  0.981712   1.0  0.177443  1.0  0.999997  0.428971  0.998438   \n",
       "79165  0.821524  0.981712   1.0  0.177443  1.0  0.999996  0.558171  0.998496   \n",
       "20549  0.826017  0.981712   1.0  0.177443  1.0  0.999999  0.329137  0.998824   \n",
       "...         ...       ...   ...       ...  ...       ...       ...       ...   \n",
       "39846  0.821232  0.927554   1.0  0.177443  1.0  0.996391  0.342791  0.998106   \n",
       "45178  0.740471  0.808703   1.0  0.334974  1.0  0.930795  0.826558  0.888832   \n",
       "25910  0.999317  0.999665   1.0  0.221467  1.0  1.000000  0.672632  0.999148   \n",
       "4089   0.826035  0.981712   1.0  0.177443  1.0  0.999995  0.308658  0.999101   \n",
       "69547  0.997685  0.999665   1.0  0.221467  1.0  1.000000  0.598654  0.999439   \n",
       "\n",
       "            xgb        lr   dt  \n",
       "82131  0.997781  1.000000  1.0  \n",
       "69866  0.996623  0.999471  1.0  \n",
       "20341  0.996581  0.999967  1.0  \n",
       "79165  0.996417  0.983103  1.0  \n",
       "20549  0.996581  0.999908  1.0  \n",
       "...         ...       ...  ...  \n",
       "39846  0.972167  0.950086  1.0  \n",
       "45178  0.899891  0.754910  1.0  \n",
       "25910  0.997881  1.000000  1.0  \n",
       "4089   0.997243  0.998960  1.0  \n",
       "69547  0.994601  0.999999  1.0  \n",
       "\n",
       "[24894 rows x 11 columns]"
      ]
     },
     "execution_count": 199,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_01"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decision tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0    2.0    3.0    4.0    5.0      6.0\n",
      "0.0  2212.0     2.0    0.0    0.0    0.0    0.0      0.0\n",
      "1.0     0.0  5499.0    0.0    0.0    1.0    0.0      0.0\n",
      "2.0     0.0     0.0  182.0    0.0    0.0    0.0      0.0\n",
      "3.0     0.0     0.0    0.0  200.0    0.0    0.0      0.0\n",
      "4.0     0.0     2.0    0.0    0.0  185.0    0.0      0.0\n",
      "5.0     0.0     0.0    0.0    0.0    0.0  181.0      0.0\n",
      "6.0     1.0     1.0    0.0    0.0    0.0    0.0  16428.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9997188077448381\n",
      "Precision total:  0.9990376214855711\n",
      "Recall total:  0.9982997033914722\n",
      "F1 total:  0.9986676172135805\n",
      "BACC total:  0.9982997033914722\n",
      "MCC total:  0.9994459333565786\n",
      "0.1895890235900879\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "start = time.time()\n",
    "\n",
    "# Create a Decision Tree Classifier\n",
    "dt_classifier = DecisionTreeClassifier(random_state=42)\n",
    "# Train the classifier on the training data\n",
    "dt_classifier.fit(X_train_01, y_train_01)\n",
    "# Make predictions on the test data\n",
    "preds_dt = dt_classifier.predict(X_test_01)\n",
    "# Evaluate the accuracy of the model\n",
    "preds_dt_prob = dt_classifier.predict_proba(X_test_01)\n",
    "\n",
    "\n",
    "pred_label = preds_dt\n",
    "name = 'dt'\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01,time_taken)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "\n",
    "globals()[f\"{name}_time_00\"] = time_taken\n",
    "print(time_taken)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'a' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-201-3f786850e387>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0ma\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'a' is not defined"
     ]
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Voting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        dt  cat  xgb  knn  mlp  ensemble\n",
      "40916  1.0  1.0  1.0  1.0  1.0         1\n",
      "62379  6.0  6.0  6.0  6.0  6.0         6\n",
      "44703  6.0  6.0  6.0  6.0  6.0         6\n",
      "58957  1.0  1.0  1.0  1.0  1.0         1\n",
      "66536  1.0  1.0  1.0  1.0  1.0         1\n",
      "...    ...  ...  ...  ...  ...       ...\n",
      "29534  6.0  6.0  6.0  6.0  6.0         6\n",
      "19243  6.0  6.0  6.0  6.0  6.0         6\n",
      "25373  1.0  1.0  1.0  1.0  1.0         1\n",
      "39314  6.0  6.0  6.0  6.0  6.0         6\n",
      "57602  6.0  6.0  6.0  6.0  6.0         6\n",
      "\n",
      "[24894 rows x 6 columns]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "confusion_metrics() missing 1 required positional argument: 'time_taken'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-180-ee3394a78f06>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     29\u001b[0m     \u001b[0;31m#testing metrics def\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m     \u001b[0mname\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'voting'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 31\u001b[0;31m     \u001b[0mmetrics\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconfusion_metrics\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpred_label\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test_01\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     32\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m     \u001b[0mend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: confusion_metrics() missing 1 required positional argument: 'time_taken'"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "    \n",
    "if pick_prob == 0:\n",
    "    # Voting start\n",
    "\n",
    "    import pandas as pd\n",
    "    from scipy.stats import mode\n",
    "\n",
    "    # Assuming 'df' is your original DataFrame with columns 'dnn', 'rf', 'lgbm', 'ada', 'knn', 'mlp', 'svm', 'label'\n",
    "    df = X_test_01\n",
    "    # Extract predictions columns\n",
    "    \n",
    "    # predictions = df[['dnn', 'rf', 'lgbm', 'ada', 'knn', 'mlp', 'svm','cat','xgb']]\n",
    "        # selected_columns = df.loc[:, ~df.columns.isin(['rf'])]\n",
    "    predictions = df.loc[:, ~df.columns.isin(['label'])] #df[column_features]\n",
    "\n",
    "    # Use the mode function along axis 1 to get the most common prediction for each row\n",
    "    ensemble_predictions, _ = mode(predictions.values, axis=1)\n",
    "\n",
    "    # Add the ensemble predictions to the DataFrame\n",
    "    df['ensemble'] = ensemble_predictions.astype(int)\n",
    "\n",
    "    # Display the DataFrame with ensemble predictions\n",
    "    print(df)\n",
    "\n",
    "    pred_label = df ['ensemble'].values\n",
    "    df.pop('ensemble')\n",
    "\n",
    "    #testing metrics def\n",
    "    name = 'voting'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n",
    "   \n",
    "else:\n",
    "    name = 'voting'\n",
    "    globals()[f\"{name}_acc_01\"] = 0\n",
    "    globals()[f\"{name}_pre_01\"] = 0\n",
    "    globals()[f\"{name}_rec_01\"] = 0\n",
    "    globals()[f\"{name}_f1_01\"] = 0\n",
    "    globals()[f\"{name}_bacc_01\"] = 0\n",
    "    globals()[f\"{name}_mcc_01\"] = 0\n",
    "    globals()[f\"{name}_time_01\"] = 9999\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9861010685305697"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "voting_acc_01\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Average"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       dnn   rf  lgbm  ada  knn  mlp  svm  cat  xgb   lr   dt  results\n",
      "39245  6.0  6.0   6.0  6.0  6.0  6.0  0.0  6.0  6.0  0.0  6.0        5\n",
      "8080   1.0  1.0   1.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0        1\n",
      "21656  6.0  6.0   5.0  6.0  5.0  5.0  6.0  5.0  5.0  5.0  5.0        5\n",
      "59562  6.0  6.0   6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0        6\n",
      "13280  6.0  0.0   0.0  6.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0        1\n",
      "...    ...  ...   ...  ...  ...  ...  ...  ...  ...  ...  ...      ...\n",
      "13853  6.0  6.0   6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0        6\n",
      "70893  6.0  6.0   6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0        6\n",
      "23766  6.0  0.0   0.0  6.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0        1\n",
      "29836  6.0  0.0   0.0  6.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0        1\n",
      "79643  6.0  6.0   6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0        6\n",
      "\n",
      "[24894 rows x 12 columns]\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0    2.0    3.0    4.0      5.0  6.0\n",
      "0.0  2062.0   192.0    9.0    1.0    2.0      1.0  0.0\n",
      "1.0  3626.0  1417.0  302.0   46.0    7.0      1.0  0.0\n",
      "2.0     0.0     0.0   52.0  107.0   17.0      1.0  0.0\n",
      "3.0     0.0     0.0    0.0  196.0    1.0      0.0  0.0\n",
      "4.0     0.0     0.0    1.0    2.0  199.0      1.0  0.0\n",
      "5.0     0.0     0.0    0.0    0.0  207.0      7.0  0.0\n",
      "6.0     3.0     9.0   17.0  286.0  609.0  15513.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.777215393267454\n",
      "Precision total:  0.2626041453402227\n",
      "Recall total:  0.37036187880043636\n",
      "F1 total:  0.2798631608312418\n",
      "BACC total:  0.37036187880043636\n",
      "MCC total:  0.594086433109225\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "# if pick_prob == 0:\n",
    "if 0 == 0:\n",
    "    # Average start\n",
    "\n",
    "    import pandas as pd\n",
    "    from scipy.stats import mode\n",
    "\n",
    "    # Assuming 'df' is your original DataFrame with columns 'dnn', 'rf', 'lgbm', 'ada', 'knn', 'mlp', 'svm', 'label'\n",
    "    df = X_test_01\n",
    "    predictions = df.loc[:, ~df.columns.isin(['label'])] #df[column_features]\n",
    "\n",
    "   \n",
    "\n",
    "    column_sums = df.sum(axis=1)\n",
    "    row_average = df.mean(axis=1)\n",
    "\n",
    "    # Approximate the result to the closest integer\n",
    "    rounded_average = row_average.round().astype(int)\n",
    "\n",
    "    # print(rounded_average)\n",
    "\n",
    "    df['results'] = rounded_average\n",
    "    print(df)\n",
    " \n",
    "    pred_label = df ['results'].values\n",
    "\n",
    "    # pred_label = df ['ensemble'].values\n",
    "    # df.pop('ensemble')\n",
    "    df.pop('results')\n",
    "\n",
    "    # df.pop('column_sums')\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    name = 'avg'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    \n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Weighed Average"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "confusion_metrics() takes 3 positional arguments but 4 were given",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-121-7b9b13629814>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0mpred_label\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpreds_dt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0mname\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'dt'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m \u001b[0mmetrics\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconfusion_metrics\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpred_label\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test_01\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtime_taken\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0mAcc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: confusion_metrics() takes 3 positional arguments but 4 were given"
     ]
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "start = time.time()\n",
    "\n",
    "# Create a Decision Tree Classifier\n",
    "dt_classifier = DecisionTreeClassifier(random_state=42)\n",
    "# Train the classifier on the training data\n",
    "dt_classifier.fit(X_train_01, y_train_01)\n",
    "# Make predictions on the test data\n",
    "preds_dt = dt_classifier.predict(X_test_01)\n",
    "# Evaluate the accuracy of the model\n",
    "preds_dt_prob = dt_classifier.predict_proba(X_test_01)\n",
    "end = time.time()\n",
    "\n",
    "time_taken = end - start\n",
    "pred_label = preds_dt\n",
    "name = 'dt'\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01,time_taken)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "\n",
    "globals()[f\"{name}_time_00\"] = time_taken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# column_features\n",
    "# move this up with column_features\n",
    "\n",
    "#important update this as you need to select the important features,\n",
    "#  the left is the least important while the right is the most important\n",
    "# needs automation\n",
    "\n",
    "# if pick_prob == 1:\n",
    "#     column_features = column_features1\n",
    "# else: column_features = column_features0\n",
    "\n",
    "if pick_prob == 1 and feature_selection_bit == 1:\n",
    "    column_features = column_features1\n",
    "\n",
    "elif pick_prob == 0 and feature_selection_bit == 1: \n",
    "    column_features = column_features0\n",
    "\n",
    "else: None\n",
    "feature_selection_columns_in_order_of_importance = column_features[:-1]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]\n",
      "[0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]\n",
      "            dnn        rf      lgbm       ada  knn       mlp       svm  \\\n",
      "40855  0.834825  0.957761  1.000000  0.177443  1.0  1.000000  0.363414   \n",
      "21065  0.159294  0.790884  0.000000  0.199597  0.4  0.675353  0.264438   \n",
      "66210  0.294455  0.974561  0.999974  0.170730  1.0  0.999978  0.036691   \n",
      "14377  0.633673  0.431216  0.999059  0.177443  1.0  0.185908  0.453513   \n",
      "66628  0.573383  0.804438  1.000000  0.177443  1.0  0.920625  0.292132   \n",
      "...         ...       ...       ...       ...  ...       ...       ...   \n",
      "34903  0.813060  0.981712  1.000000  0.177443  1.0  1.000000  0.350415   \n",
      "34454  0.615815  0.761394  0.999755  0.177443  1.0  0.761677  0.264642   \n",
      "26745  0.786504  0.970401  1.000000  0.226642  1.0  1.000000  0.355677   \n",
      "13252  0.678052  0.898387  1.000000  0.226642  1.0  0.999874  0.341117   \n",
      "35975  0.220471  0.974561  1.000000  0.170730  1.0  0.999719  0.486414   \n",
      "\n",
      "            cat       xgb        lr   dt  \n",
      "40855  0.994199  0.982694  0.999231  1.0  \n",
      "21065  0.997573  0.975601  0.130623  1.0  \n",
      "66210  0.994867  0.998961  0.980950  1.0  \n",
      "14377  0.973849  0.960973  0.141591  1.0  \n",
      "66628  0.995728  0.883709  0.837210  1.0  \n",
      "...         ...       ...       ...  ...  \n",
      "34903  0.998846  0.996581  0.999793  1.0  \n",
      "34454  0.972132  0.976820  0.562708  1.0  \n",
      "26745  0.998774  0.997656  0.996906  1.0  \n",
      "13252  0.988753  0.976521  0.960572  1.0  \n",
      "35975  0.995729  0.998961  0.956922  1.0  \n",
      "\n",
      "[24894 rows x 11 columns]\n",
      "40855    0.881538\n",
      "21065    0.616664\n",
      "66210    0.845292\n",
      "14377    0.661660\n",
      "66628    0.823043\n",
      "           ...   \n",
      "34903    0.883259\n",
      "34454    0.770424\n",
      "26745    0.885986\n",
      "13252    0.872781\n",
      "35975    0.890508\n",
      "Length: 24894, dtype: float64\n",
      "40855    1\n",
      "21065    1\n",
      "66210    1\n",
      "14377    1\n",
      "66628    1\n",
      "        ..\n",
      "34903    1\n",
      "34454    1\n",
      "26745    1\n",
      "13252    1\n",
      "35975    1\n",
      "Length: 24894, dtype: int64\n",
      "            dnn        rf      lgbm       ada  knn       mlp       svm  \\\n",
      "40855  0.834825  0.957761  1.000000  0.177443  1.0  1.000000  0.363414   \n",
      "21065  0.159294  0.790884  0.000000  0.199597  0.4  0.675353  0.264438   \n",
      "66210  0.294455  0.974561  0.999974  0.170730  1.0  0.999978  0.036691   \n",
      "14377  0.633673  0.431216  0.999059  0.177443  1.0  0.185908  0.453513   \n",
      "66628  0.573383  0.804438  1.000000  0.177443  1.0  0.920625  0.292132   \n",
      "...         ...       ...       ...       ...  ...       ...       ...   \n",
      "34903  0.813060  0.981712  1.000000  0.177443  1.0  1.000000  0.350415   \n",
      "34454  0.615815  0.761394  0.999755  0.177443  1.0  0.761677  0.264642   \n",
      "26745  0.786504  0.970401  1.000000  0.226642  1.0  1.000000  0.355677   \n",
      "13252  0.678052  0.898387  1.000000  0.226642  1.0  0.999874  0.341117   \n",
      "35975  0.220471  0.974561  1.000000  0.170730  1.0  0.999719  0.486414   \n",
      "\n",
      "            cat       xgb        lr   dt  results  \n",
      "40855  0.994199  0.982694  0.999231  1.0        1  \n",
      "21065  0.997573  0.975601  0.130623  1.0        1  \n",
      "66210  0.994867  0.998961  0.980950  1.0        1  \n",
      "14377  0.973849  0.960973  0.141591  1.0        1  \n",
      "66628  0.995728  0.883709  0.837210  1.0        1  \n",
      "...         ...       ...       ...  ...      ...  \n",
      "34903  0.998846  0.996581  0.999793  1.0        1  \n",
      "34454  0.972132  0.976820  0.562708  1.0        1  \n",
      "26745  0.998774  0.997656  0.996906  1.0        1  \n",
      "13252  0.988753  0.976521  0.960572  1.0        1  \n",
      "35975  0.995729  0.998961  0.956922  1.0        1  \n",
      "\n",
      "[24894 rows x 12 columns]\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "      0.0      1.0  2.0  3.0  4.0  5.0  6.0\n",
      "0.0   6.0   2331.0  0.0  0.0  0.0  0.0  0.0\n",
      "1.0   4.0   5477.0  0.0  0.0  0.0  0.0  0.0\n",
      "2.0  19.0    171.0  0.0  0.0  0.0  0.0  0.0\n",
      "3.0   1.0    188.0  0.0  0.0  0.0  0.0  0.0\n",
      "4.0   7.0    160.0  0.0  0.0  0.0  0.0  0.0\n",
      "5.0   2.0    215.0  0.0  0.0  0.0  0.0  0.0\n",
      "6.0  24.0  16289.0  0.0  0.0  0.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.220253876436089\n",
      "Precision total:  0.04511559365802621\n",
      "Recall total:  0.14311965718025021\n",
      "F1 total:  0.05233929042717641\n",
      "BACC total:  0.14311965718025021\n",
      "MCC total:  0.00785862082275999\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "# if pick_prob == 0:\n",
    "if 0 == 0:\n",
    "    # Average start\n",
    "\n",
    "    import pandas as pd\n",
    "    from scipy.stats import mode\n",
    "\n",
    "    # Assuming 'df' is your original DataFrame with columns 'dnn', 'rf', 'lgbm', 'ada', 'knn', 'mlp', 'svm', 'label'\n",
    "    # df = X_test_01\n",
    "    df = X_test_01[feature_selection_columns_in_order_of_importance]\n",
    "    # Extract predictions columns\n",
    "    \n",
    "    # predictions = df[['dnn', 'rf', 'lgbm', 'ada', 'knn', 'mlp', 'svm','cat','xgb']]\n",
    "        # selected_columns = df.loc[:, ~df.columns.isin(['rf'])]\n",
    "    predictions = df.loc[:, ~df.columns.isin(['label'])] #df[column_features]\n",
    "\n",
    "    # weight\n",
    "    weights_values = []\n",
    "\n",
    "    # linear weight distribution\n",
    "    for i in range(0,len(~df.columns.isin(['label']))):\n",
    "        weights_values.append(i/(len(~df.columns.isin(['label']))-1))\n",
    "    print(weights_values)\n",
    "    # weights_values = [10,3,2,2.3]\n",
    "    print(weights_values)\n",
    "    print(df)\n",
    "    weighted_average = df.multiply(weights_values).sum(axis=1) / sum(weights_values)\n",
    "    print(weighted_average)\n",
    "    # Approximate the result to the closest integer\n",
    "    rounded_weighted_average = weighted_average.round().astype(int)\n",
    "\n",
    "    print(rounded_weighted_average)\n",
    "\n",
    "    # print(rounded_average)\n",
    "\n",
    "    df['results'] = rounded_weighted_average\n",
    "    print(df)\n",
    " \n",
    "    pred_label = df ['results'].values\n",
    "\n",
    "    # pred_label = df ['ensemble'].values\n",
    "    # df.pop('ensemble')\n",
    "    df.pop('results')\n",
    "\n",
    "    # df.pop('column_sums')\n",
    "\n",
    "    #testing metrics def\n",
    "    name = 'weighed_avg'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging  with DT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0    2.0    3.0    4.0    5.0      6.0\n",
      "0.0  2337.0     0.0    0.0    0.0    0.0    0.0      0.0\n",
      "1.0     0.0  5481.0    0.0    0.0    0.0    0.0      0.0\n",
      "2.0     0.0     0.0  190.0    0.0    0.0    0.0      0.0\n",
      "3.0     0.0     0.0    0.0  189.0    0.0    0.0      0.0\n",
      "4.0     0.0     0.0    0.0    0.0  167.0    0.0      0.0\n",
      "5.0     0.0     0.0    0.0    0.0    0.0  217.0      0.0\n",
      "6.0     0.0     0.0    0.0    0.0    0.0    0.0  16313.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  1.0\n",
      "Precision total:  1.0\n",
      "Recall total:  1.0\n",
      "F1 total:  1.0\n",
      "BACC total:  1.0\n",
      "MCC total:  1.0\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "start = time.time()\n",
    "base_classifier = DecisionTreeClassifier(random_state=42)\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test_01, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_dt'\n",
    "pred_label = y_pred\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging  with SVM\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0    2.0  3.0    4.0    5.0      6.0\n",
      "0.0  2309.0    13.0    2.0  0.0    0.0    0.0     13.0\n",
      "1.0    26.0  1666.0   17.0  0.0    3.0   43.0   3726.0\n",
      "2.0     3.0     0.0  181.0  1.0    5.0    0.0      0.0\n",
      "3.0     0.0   186.0    0.0  1.0    2.0    0.0      0.0\n",
      "4.0     0.0     0.0   24.0  0.0  138.0    0.0      5.0\n",
      "5.0    73.0     0.0   24.0  0.0    0.0  119.0      1.0\n",
      "6.0    19.0   440.0    0.0  0.0    0.0    0.0  15854.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.8141720896601591\n",
      "Precision total:  0.7683914571026147\n",
      "Recall total:  0.6566425538440323\n",
      "F1 total:  0.6601058778687535\n",
      "BACC total:  0.6566425538440323\n",
      "MCC total:  0.6232053352922977\n"
     ]
    }
   ],
   "source": [
    "## bagging  with SVM\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "# Instantiate the SGDClassifier with additional hyperparameters\n",
    "svm_01 = SGDClassifier(\n",
    "    loss='hinge',           # hinge loss for linear SVM\n",
    "    penalty='l2',           # L2 regularization to prevent overfitting\n",
    "    alpha=1e-4,             # Learning rate (small value for fine-grained updates)\n",
    "    max_iter=1000,          # Number of passes over the training data\n",
    "    random_state=42,        # Seed for reproducible results\n",
    "    learning_rate='optimal' # Automatically adjusts the learning rate based on the training data\n",
    ")\n",
    "\n",
    "# # Define the base classifier (Decision Tree in this case)\n",
    "base_classifier = svm_01\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_svm'\n",
    "pred_label = y_pred\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging with DNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from tensorflow.keras.models import Sequential\n",
    "# from tensorflow.keras.layers import Dense\n",
    "\n",
    "# #Model Parameters\n",
    "# dropout_rate = 0.2\n",
    "# nodes = 3\n",
    "# out_layer = 5\n",
    "# optimizer='adam'\n",
    "# loss='sparse_categorical_crossentropy'\n",
    "# epochs=100\n",
    "# batch_size=128\n",
    "\n",
    "\n",
    "# num_columns = X_train_01.shape[1]\n",
    "\n",
    "# dnn_01 = tf.keras.Sequential()\n",
    "\n",
    "# # Input layer\n",
    "# dnn_01.add(tf.keras.Input(shape=(num_columns,)))\n",
    "\n",
    "# # Dense layers with dropout\n",
    "# dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# # Output layer\n",
    "# # dnn_01.add(tf.keras.layers.Dense(out_layer))\n",
    "\n",
    "# dnn_01.add(tf.keras.layers.Dense(out_layer, activation='softmax'))\n",
    "\n",
    "\n",
    "# dnn_01.compile(optimizer=optimizer, loss=loss,metrics=['accuracy'])\n",
    "\n",
    "# base_classifier = dnn_01\n",
    "\n",
    "# # Define the BaggingClassifier\n",
    "# bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# # Train the BaggingClassifier\n",
    "# bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# # Make predictions on the test set\n",
    "# y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "# # Evaluate accuracy\n",
    "# # accuracy = accuracy_score(y_test_01, y_pred)\n",
    "# # print(f'Accuracy: {accuracy}')\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('Bagging with DNN', file = f)\n",
    "\n",
    "\n",
    "# print('---------------------------------------------------------------------------------')\n",
    "# print('CONFUSION MATRIX')\n",
    "# print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "# pred_label = y_pred\n",
    "\n",
    "# confusion_matrix = pd.crosstab(y_test_01, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "# all_unique_values = sorted(set(pred_label) | set(y_test_01))\n",
    "# z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "# rows, cols = confusion_matrix.shape\n",
    "# z[:rows, :cols] = confusion_matrix\n",
    "# confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "# # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "# # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "# print(confusion_matrix)\n",
    "# with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "# FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "# FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "# TP = np.diag(confusion_matrix)\n",
    "# TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "# TP_total = sum(TP)\n",
    "# TN_total = sum(TN)\n",
    "# FP_total = sum(FP)\n",
    "# FN_total = sum(FN)\n",
    "\n",
    "# TP_total = np.array(TP_total,dtype=np.float64)\n",
    "# TN_total = np.array(TN_total,dtype=np.float64)\n",
    "# FP_total = np.array(FP_total,dtype=np.float64)\n",
    "# FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "# #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "# print('---------------------------------------------------------------------------------')\n",
    "# print('METRICS')\n",
    "# print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "# Acc = accuracy_score(y_test_01, pred_label)\n",
    "# Precision = precision_score(y_test_01, pred_label, average='macro')\n",
    "# Recall = recall_score(y_test_01, pred_label, average='macro')\n",
    "# F1 =  f1_score(y_test_01, pred_label, average='macro')\n",
    "# BACC = balanced_accuracy_score(y_test_01, pred_label)\n",
    "# MCC = matthews_corrcoef(y_test_01, pred_label)\n",
    "\n",
    "\n",
    "# bag_dnn_acc_01 = Acc\n",
    "# bag_dnn_pre_01 = Precision\n",
    "# bag_dnn_rec_01 = Recall\n",
    "# bag_dnn_f1_01 = F1\n",
    "# bag_dnn_bacc_01 = BACC\n",
    "# bag_dnn_mcc_01 = MCC\n",
    "# # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "# print('Accuracy total: ', Acc)\n",
    "# print('Precision total: ', Precision )\n",
    "# print('Recall total: ', Recall )\n",
    "# print('F1 total: ', F1 )\n",
    "# print('BACC total: ', BACC)\n",
    "# print('MCC total: ', MCC)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging with MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0    2.0    3.0    4.0    5.0      6.0\n",
      "0.0  2332.0     3.0    1.0    0.0    0.0    0.0      1.0\n",
      "1.0     4.0  5436.0    0.0    0.0    0.0    0.0     41.0\n",
      "2.0     0.0     0.0  185.0    0.0    4.0    1.0      0.0\n",
      "3.0     0.0     0.0    0.0  189.0    0.0    0.0      0.0\n",
      "4.0     0.0     2.0    5.0    0.0  159.0    1.0      0.0\n",
      "5.0     0.0     0.0    8.0    0.0    7.0  202.0      0.0\n",
      "6.0     4.0    15.0    0.0    0.0    0.0    0.0  16294.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9961034787498996\n",
      "Precision total:  0.977926133700173\n",
      "Recall total:  0.977877314855073\n",
      "F1 total:  0.9776865567317891\n",
      "BACC total:  0.977877314855073\n",
      "MCC total:  0.9924019643670448\n"
     ]
    }
   ],
   "source": [
    "from sklearn.neural_network import MLPClassifier\n",
    "start = time.time()\n",
    "\n",
    "# create MLPClassifier instance\n",
    "mlp_01 = MLPClassifier(hidden_layer_sizes=(100,), max_iter=200, random_state=1)\n",
    "\n",
    "base_classifier = mlp_01\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test_01, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_mlp'\n",
    "pred_label = y_pred\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging knn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0    2.0    3.0    4.0    5.0      6.0\n",
      "0.0  2328.0     3.0    1.0    0.0    2.0    1.0      2.0\n",
      "1.0     4.0  5431.0    6.0    0.0    0.0    0.0     40.0\n",
      "2.0     1.0     2.0  183.0    0.0    3.0    1.0      0.0\n",
      "3.0     1.0     0.0    2.0  186.0    0.0    0.0      0.0\n",
      "4.0     1.0     0.0    5.0    0.0  161.0    0.0      0.0\n",
      "5.0     0.0     0.0    4.0    0.0    3.0  210.0      0.0\n",
      "6.0     3.0    11.0    0.0    0.0    0.0    0.0  16299.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9961436490720655\n",
      "Precision total:  0.9776988202835447\n",
      "Recall total:  0.9807524207620769\n",
      "F1 total:  0.9790852914473641\n",
      "BACC total:  0.9807524207620769\n",
      "MCC total:  0.9924807696327641\n"
     ]
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "knn_01=KNeighborsClassifier(n_neighbors = 5)\n",
    "start = time.time()\n",
    "\n",
    "base_classifier = knn_01\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test_01, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_knn'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging LogRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining baggin Logistic Regression Model\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0    2.0    3.0    4.0    5.0      6.0\n",
      "0.0  2296.0    22.0    3.0    0.0    0.0    1.0     15.0\n",
      "1.0     9.0  3487.0   15.0    0.0    2.0   26.0   1942.0\n",
      "2.0     0.0     0.0  177.0    1.0    5.0    7.0      0.0\n",
      "3.0     0.0     2.0    0.0  185.0    2.0    0.0      0.0\n",
      "4.0     0.0     0.0   28.0    0.0  138.0    0.0      1.0\n",
      "5.0    54.0     0.0   18.0    0.0    3.0  141.0      1.0\n",
      "6.0    16.0   990.0    0.0    0.0    0.0    0.0  15307.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.8729412709889933\n",
      "Precision total:  0.8689673383177559\n",
      "Recall total:  0.8490738747811429\n",
      "F1 total:  0.8547077208519952\n",
      "BACC total:  0.8490738747811429\n",
      "MCC total:  0.7468628168305194\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "start = time.time()\n",
    "\n",
    "#Logistic Regression\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining baggin Logistic Regression Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "logreg_01 = LogisticRegression()\n",
    "\n",
    "\n",
    "base_classifier = logreg_01\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test_01, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_lr'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging ADA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0    2.0    3.0    4.0    5.0      6.0\n",
      "0.0  1669.0   668.0    0.0    0.0    0.0    0.0      0.0\n",
      "1.0     0.0  2250.0    0.0    0.0    0.0    0.0   3231.0\n",
      "2.0     0.0     0.0  190.0    0.0    0.0    0.0      0.0\n",
      "3.0     0.0     0.0    0.0  189.0    0.0    0.0      0.0\n",
      "4.0     0.0     2.0    0.0    0.0  165.0    0.0      0.0\n",
      "5.0     0.0     0.0    0.0    0.0   75.0  142.0      0.0\n",
      "6.0     1.0  4333.0    0.0    0.0    0.0    0.0  11979.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.6661846228006748\n",
      "Precision total:  0.8263845177507748\n",
      "Recall total:  0.7859138057893231\n",
      "F1 total:  0.7926200676309945\n",
      "BACC total:  0.7859138057893231\n",
      "MCC total:  0.3717249912461424\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "import time\n",
    "ada = AdaBoostClassifier(n_estimators=50, learning_rate=1.0)\n",
    "\n",
    "base_classifier = ada\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_ada'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging CAT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 1.4084011\ttotal: 73.2ms\tremaining: 7.25s\n",
      "1:\tlearn: 1.1322868\ttotal: 88.9ms\tremaining: 4.36s\n",
      "2:\tlearn: 0.9425689\ttotal: 102ms\tremaining: 3.3s\n",
      "3:\tlearn: 0.8016113\ttotal: 114ms\tremaining: 2.73s\n",
      "4:\tlearn: 0.6911454\ttotal: 124ms\tremaining: 2.36s\n",
      "5:\tlearn: 0.6015939\ttotal: 135ms\tremaining: 2.11s\n",
      "6:\tlearn: 0.5260637\ttotal: 145ms\tremaining: 1.93s\n",
      "7:\tlearn: 0.4609664\ttotal: 155ms\tremaining: 1.78s\n",
      "8:\tlearn: 0.4087234\ttotal: 165ms\tremaining: 1.66s\n",
      "9:\tlearn: 0.3630580\ttotal: 175ms\tremaining: 1.57s\n",
      "10:\tlearn: 0.3243841\ttotal: 184ms\tremaining: 1.49s\n",
      "11:\tlearn: 0.2901251\ttotal: 194ms\tremaining: 1.42s\n",
      "12:\tlearn: 0.2588894\ttotal: 203ms\tremaining: 1.36s\n",
      "13:\tlearn: 0.2326609\ttotal: 213ms\tremaining: 1.31s\n",
      "14:\tlearn: 0.2095431\ttotal: 223ms\tremaining: 1.26s\n",
      "15:\tlearn: 0.1889052\ttotal: 233ms\tremaining: 1.22s\n",
      "16:\tlearn: 0.1706754\ttotal: 243ms\tremaining: 1.19s\n",
      "17:\tlearn: 0.1542232\ttotal: 252ms\tremaining: 1.15s\n",
      "18:\tlearn: 0.1399384\ttotal: 262ms\tremaining: 1.12s\n",
      "19:\tlearn: 0.1265739\ttotal: 272ms\tremaining: 1.09s\n",
      "20:\tlearn: 0.1146990\ttotal: 282ms\tremaining: 1.06s\n",
      "21:\tlearn: 0.1042224\ttotal: 292ms\tremaining: 1.03s\n",
      "22:\tlearn: 0.0944582\ttotal: 302ms\tremaining: 1.01s\n",
      "23:\tlearn: 0.0860468\ttotal: 311ms\tremaining: 986ms\n",
      "24:\tlearn: 0.0780779\ttotal: 321ms\tremaining: 964ms\n",
      "25:\tlearn: 0.0707103\ttotal: 331ms\tremaining: 942ms\n",
      "26:\tlearn: 0.0644267\ttotal: 341ms\tremaining: 922ms\n",
      "27:\tlearn: 0.0585795\ttotal: 351ms\tremaining: 902ms\n",
      "28:\tlearn: 0.0535260\ttotal: 360ms\tremaining: 882ms\n",
      "29:\tlearn: 0.0485706\ttotal: 370ms\tremaining: 864ms\n",
      "30:\tlearn: 0.0445884\ttotal: 380ms\tremaining: 847ms\n",
      "31:\tlearn: 0.0407584\ttotal: 390ms\tremaining: 829ms\n",
      "32:\tlearn: 0.0373453\ttotal: 401ms\tremaining: 814ms\n",
      "33:\tlearn: 0.0344175\ttotal: 411ms\tremaining: 797ms\n",
      "34:\tlearn: 0.0316935\ttotal: 420ms\tremaining: 780ms\n",
      "35:\tlearn: 0.0290227\ttotal: 430ms\tremaining: 764ms\n",
      "36:\tlearn: 0.0267516\ttotal: 440ms\tremaining: 749ms\n",
      "37:\tlearn: 0.0246219\ttotal: 449ms\tremaining: 733ms\n",
      "38:\tlearn: 0.0226583\ttotal: 459ms\tremaining: 717ms\n",
      "39:\tlearn: 0.0209583\ttotal: 468ms\tremaining: 702ms\n",
      "40:\tlearn: 0.0193936\ttotal: 478ms\tremaining: 687ms\n",
      "41:\tlearn: 0.0180939\ttotal: 487ms\tremaining: 673ms\n",
      "42:\tlearn: 0.0167947\ttotal: 497ms\tremaining: 659ms\n",
      "43:\tlearn: 0.0154407\ttotal: 507ms\tremaining: 645ms\n",
      "44:\tlearn: 0.0143120\ttotal: 517ms\tremaining: 631ms\n",
      "45:\tlearn: 0.0133922\ttotal: 526ms\tremaining: 618ms\n",
      "46:\tlearn: 0.0124902\ttotal: 536ms\tremaining: 605ms\n",
      "47:\tlearn: 0.0116721\ttotal: 545ms\tremaining: 591ms\n",
      "48:\tlearn: 0.0110423\ttotal: 556ms\tremaining: 579ms\n",
      "49:\tlearn: 0.0104426\ttotal: 566ms\tremaining: 566ms\n",
      "50:\tlearn: 0.0097854\ttotal: 576ms\tremaining: 553ms\n",
      "51:\tlearn: 0.0092664\ttotal: 585ms\tremaining: 540ms\n",
      "52:\tlearn: 0.0087914\ttotal: 595ms\tremaining: 527ms\n",
      "53:\tlearn: 0.0083242\ttotal: 604ms\tremaining: 515ms\n",
      "54:\tlearn: 0.0078936\ttotal: 614ms\tremaining: 502ms\n",
      "55:\tlearn: 0.0075275\ttotal: 623ms\tremaining: 489ms\n",
      "56:\tlearn: 0.0070546\ttotal: 632ms\tremaining: 477ms\n",
      "57:\tlearn: 0.0066852\ttotal: 642ms\tremaining: 465ms\n",
      "58:\tlearn: 0.0064030\ttotal: 651ms\tremaining: 453ms\n",
      "59:\tlearn: 0.0060811\ttotal: 661ms\tremaining: 440ms\n",
      "60:\tlearn: 0.0058595\ttotal: 670ms\tremaining: 428ms\n",
      "61:\tlearn: 0.0056193\ttotal: 679ms\tremaining: 416ms\n",
      "62:\tlearn: 0.0052924\ttotal: 691ms\tremaining: 406ms\n",
      "63:\tlearn: 0.0051195\ttotal: 701ms\tremaining: 394ms\n",
      "64:\tlearn: 0.0048802\ttotal: 712ms\tremaining: 383ms\n",
      "65:\tlearn: 0.0047163\ttotal: 721ms\tremaining: 371ms\n",
      "66:\tlearn: 0.0045538\ttotal: 731ms\tremaining: 360ms\n",
      "67:\tlearn: 0.0044044\ttotal: 740ms\tremaining: 348ms\n",
      "68:\tlearn: 0.0041660\ttotal: 750ms\tremaining: 337ms\n",
      "69:\tlearn: 0.0039728\ttotal: 760ms\tremaining: 326ms\n",
      "70:\tlearn: 0.0037640\ttotal: 771ms\tremaining: 315ms\n",
      "71:\tlearn: 0.0036757\ttotal: 780ms\tremaining: 303ms\n",
      "72:\tlearn: 0.0035043\ttotal: 790ms\tremaining: 292ms\n",
      "73:\tlearn: 0.0033752\ttotal: 800ms\tremaining: 281ms\n",
      "74:\tlearn: 0.0032768\ttotal: 809ms\tremaining: 270ms\n",
      "75:\tlearn: 0.0031653\ttotal: 820ms\tremaining: 259ms\n",
      "76:\tlearn: 0.0030588\ttotal: 830ms\tremaining: 248ms\n",
      "77:\tlearn: 0.0029726\ttotal: 840ms\tremaining: 237ms\n",
      "78:\tlearn: 0.0028995\ttotal: 849ms\tremaining: 226ms\n",
      "79:\tlearn: 0.0028251\ttotal: 859ms\tremaining: 215ms\n",
      "80:\tlearn: 0.0027537\ttotal: 868ms\tremaining: 204ms\n",
      "81:\tlearn: 0.0026277\ttotal: 878ms\tremaining: 193ms\n",
      "82:\tlearn: 0.0025831\ttotal: 887ms\tremaining: 182ms\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "83:\tlearn: 0.0025136\ttotal: 897ms\tremaining: 171ms\n",
      "84:\tlearn: 0.0024326\ttotal: 907ms\tremaining: 160ms\n",
      "85:\tlearn: 0.0023980\ttotal: 916ms\tremaining: 149ms\n",
      "86:\tlearn: 0.0022792\ttotal: 927ms\tremaining: 138ms\n",
      "87:\tlearn: 0.0022279\ttotal: 938ms\tremaining: 128ms\n",
      "88:\tlearn: 0.0021929\ttotal: 950ms\tremaining: 117ms\n",
      "89:\tlearn: 0.0021467\ttotal: 961ms\tremaining: 107ms\n",
      "90:\tlearn: 0.0021304\ttotal: 969ms\tremaining: 95.8ms\n",
      "91:\tlearn: 0.0020869\ttotal: 981ms\tremaining: 85.3ms\n",
      "92:\tlearn: 0.0020381\ttotal: 992ms\tremaining: 74.6ms\n",
      "93:\tlearn: 0.0019527\ttotal: 1s\tremaining: 64.1ms\n",
      "94:\tlearn: 0.0019195\ttotal: 1.02s\tremaining: 53.5ms\n",
      "95:\tlearn: 0.0018601\ttotal: 1.03s\tremaining: 42.8ms\n",
      "96:\tlearn: 0.0018289\ttotal: 1.04s\tremaining: 32.1ms\n",
      "97:\tlearn: 0.0018224\ttotal: 1.04s\tremaining: 21.3ms\n",
      "98:\tlearn: 0.0017907\ttotal: 1.06s\tremaining: 10.7ms\n",
      "99:\tlearn: 0.0017434\ttotal: 1.07s\tremaining: 0us\n",
      "0:\tlearn: 1.4081212\ttotal: 18.6ms\tremaining: 1.84s\n",
      "1:\tlearn: 1.1314599\ttotal: 38.8ms\tremaining: 1.9s\n",
      "2:\tlearn: 0.9414935\ttotal: 57.5ms\tremaining: 1.86s\n",
      "3:\tlearn: 0.8010123\ttotal: 73.1ms\tremaining: 1.75s\n",
      "4:\tlearn: 0.6916102\ttotal: 89.1ms\tremaining: 1.69s\n",
      "5:\tlearn: 0.6021998\ttotal: 103ms\tremaining: 1.62s\n",
      "6:\tlearn: 0.5265906\ttotal: 119ms\tremaining: 1.58s\n",
      "7:\tlearn: 0.4629863\ttotal: 133ms\tremaining: 1.52s\n",
      "8:\tlearn: 0.4106234\ttotal: 146ms\tremaining: 1.48s\n",
      "9:\tlearn: 0.3647578\ttotal: 160ms\tremaining: 1.44s\n",
      "10:\tlearn: 0.3257366\ttotal: 174ms\tremaining: 1.4s\n",
      "11:\tlearn: 0.2914174\ttotal: 186ms\tremaining: 1.37s\n",
      "12:\tlearn: 0.2611026\ttotal: 199ms\tremaining: 1.33s\n",
      "13:\tlearn: 0.2350065\ttotal: 212ms\tremaining: 1.3s\n",
      "14:\tlearn: 0.2108569\ttotal: 224ms\tremaining: 1.27s\n",
      "15:\tlearn: 0.1902977\ttotal: 236ms\tremaining: 1.24s\n",
      "16:\tlearn: 0.1717585\ttotal: 246ms\tremaining: 1.2s\n",
      "17:\tlearn: 0.1551314\ttotal: 256ms\tremaining: 1.17s\n",
      "18:\tlearn: 0.1405878\ttotal: 266ms\tremaining: 1.13s\n",
      "19:\tlearn: 0.1274189\ttotal: 276ms\tremaining: 1.1s\n",
      "20:\tlearn: 0.1152728\ttotal: 285ms\tremaining: 1.07s\n",
      "21:\tlearn: 0.1047083\ttotal: 295ms\tremaining: 1.04s\n",
      "22:\tlearn: 0.0952037\ttotal: 305ms\tremaining: 1.02s\n",
      "23:\tlearn: 0.0867536\ttotal: 315ms\tremaining: 998ms\n",
      "24:\tlearn: 0.0791815\ttotal: 325ms\tremaining: 975ms\n",
      "25:\tlearn: 0.0717892\ttotal: 335ms\tremaining: 953ms\n",
      "26:\tlearn: 0.0653800\ttotal: 345ms\tremaining: 932ms\n",
      "27:\tlearn: 0.0597540\ttotal: 354ms\tremaining: 910ms\n",
      "28:\tlearn: 0.0545793\ttotal: 364ms\tremaining: 891ms\n",
      "29:\tlearn: 0.0498732\ttotal: 374ms\tremaining: 872ms\n",
      "30:\tlearn: 0.0457684\ttotal: 385ms\tremaining: 857ms\n",
      "31:\tlearn: 0.0419934\ttotal: 395ms\tremaining: 840ms\n",
      "32:\tlearn: 0.0386129\ttotal: 406ms\tremaining: 823ms\n",
      "33:\tlearn: 0.0356315\ttotal: 416ms\tremaining: 807ms\n",
      "34:\tlearn: 0.0329472\ttotal: 427ms\tremaining: 793ms\n",
      "35:\tlearn: 0.0302955\ttotal: 438ms\tremaining: 778ms\n",
      "36:\tlearn: 0.0279340\ttotal: 449ms\tremaining: 764ms\n",
      "37:\tlearn: 0.0256539\ttotal: 460ms\tremaining: 751ms\n",
      "38:\tlearn: 0.0237888\ttotal: 472ms\tremaining: 738ms\n",
      "39:\tlearn: 0.0221457\ttotal: 483ms\tremaining: 724ms\n",
      "40:\tlearn: 0.0204990\ttotal: 494ms\tremaining: 710ms\n",
      "41:\tlearn: 0.0188967\ttotal: 504ms\tremaining: 696ms\n",
      "42:\tlearn: 0.0175937\ttotal: 515ms\tremaining: 683ms\n",
      "43:\tlearn: 0.0163559\ttotal: 527ms\tremaining: 671ms\n",
      "44:\tlearn: 0.0154961\ttotal: 538ms\tremaining: 657ms\n",
      "45:\tlearn: 0.0145207\ttotal: 549ms\tremaining: 645ms\n",
      "46:\tlearn: 0.0136857\ttotal: 561ms\tremaining: 633ms\n",
      "47:\tlearn: 0.0127618\ttotal: 572ms\tremaining: 620ms\n",
      "48:\tlearn: 0.0120357\ttotal: 583ms\tremaining: 607ms\n",
      "49:\tlearn: 0.0113396\ttotal: 594ms\tremaining: 594ms\n",
      "50:\tlearn: 0.0105379\ttotal: 606ms\tremaining: 582ms\n",
      "51:\tlearn: 0.0099764\ttotal: 616ms\tremaining: 569ms\n",
      "52:\tlearn: 0.0094609\ttotal: 627ms\tremaining: 556ms\n",
      "53:\tlearn: 0.0091296\ttotal: 636ms\tremaining: 542ms\n",
      "54:\tlearn: 0.0087880\ttotal: 644ms\tremaining: 527ms\n",
      "55:\tlearn: 0.0082152\ttotal: 655ms\tremaining: 515ms\n",
      "56:\tlearn: 0.0077484\ttotal: 667ms\tremaining: 503ms\n",
      "57:\tlearn: 0.0074526\ttotal: 677ms\tremaining: 491ms\n",
      "58:\tlearn: 0.0071634\ttotal: 687ms\tremaining: 477ms\n",
      "59:\tlearn: 0.0068797\ttotal: 695ms\tremaining: 463ms\n",
      "60:\tlearn: 0.0066801\ttotal: 704ms\tremaining: 450ms\n",
      "61:\tlearn: 0.0062614\ttotal: 715ms\tremaining: 438ms\n",
      "62:\tlearn: 0.0059538\ttotal: 726ms\tremaining: 426ms\n",
      "63:\tlearn: 0.0057226\ttotal: 736ms\tremaining: 414ms\n",
      "64:\tlearn: 0.0053867\ttotal: 746ms\tremaining: 402ms\n",
      "65:\tlearn: 0.0050815\ttotal: 761ms\tremaining: 392ms\n",
      "66:\tlearn: 0.0049236\ttotal: 771ms\tremaining: 380ms\n",
      "67:\tlearn: 0.0047856\ttotal: 781ms\tremaining: 367ms\n",
      "68:\tlearn: 0.0045599\ttotal: 792ms\tremaining: 356ms\n",
      "69:\tlearn: 0.0044004\ttotal: 802ms\tremaining: 344ms\n",
      "70:\tlearn: 0.0042558\ttotal: 811ms\tremaining: 331ms\n",
      "71:\tlearn: 0.0040982\ttotal: 836ms\tremaining: 325ms\n",
      "72:\tlearn: 0.0039424\ttotal: 846ms\tremaining: 313ms\n",
      "73:\tlearn: 0.0038542\ttotal: 856ms\tremaining: 301ms\n",
      "74:\tlearn: 0.0036700\ttotal: 866ms\tremaining: 289ms\n",
      "75:\tlearn: 0.0035485\ttotal: 876ms\tremaining: 277ms\n",
      "76:\tlearn: 0.0034626\ttotal: 886ms\tremaining: 265ms\n",
      "77:\tlearn: 0.0033396\ttotal: 898ms\tremaining: 253ms\n",
      "78:\tlearn: 0.0032854\ttotal: 909ms\tremaining: 242ms\n",
      "79:\tlearn: 0.0031475\ttotal: 921ms\tremaining: 230ms\n",
      "80:\tlearn: 0.0030268\ttotal: 931ms\tremaining: 218ms\n",
      "81:\tlearn: 0.0029362\ttotal: 942ms\tremaining: 207ms\n",
      "82:\tlearn: 0.0028950\ttotal: 952ms\tremaining: 195ms\n",
      "83:\tlearn: 0.0028219\ttotal: 962ms\tremaining: 183ms\n",
      "84:\tlearn: 0.0026982\ttotal: 973ms\tremaining: 172ms\n",
      "85:\tlearn: 0.0026711\ttotal: 980ms\tremaining: 159ms\n",
      "86:\tlearn: 0.0025895\ttotal: 989ms\tremaining: 148ms\n",
      "87:\tlearn: 0.0025480\ttotal: 999ms\tremaining: 136ms\n",
      "88:\tlearn: 0.0025049\ttotal: 1.01s\tremaining: 125ms\n",
      "89:\tlearn: 0.0024708\ttotal: 1.02s\tremaining: 113ms\n",
      "90:\tlearn: 0.0023637\ttotal: 1.03s\tremaining: 102ms\n",
      "91:\tlearn: 0.0023475\ttotal: 1.04s\tremaining: 90.1ms\n",
      "92:\tlearn: 0.0023282\ttotal: 1.04s\tremaining: 78.7ms\n",
      "93:\tlearn: 0.0022844\ttotal: 1.05s\tremaining: 67.4ms\n",
      "94:\tlearn: 0.0022395\ttotal: 1.07s\tremaining: 56.1ms\n",
      "95:\tlearn: 0.0021763\ttotal: 1.08s\tremaining: 44.9ms\n",
      "96:\tlearn: 0.0020898\ttotal: 1.09s\tremaining: 33.8ms\n",
      "97:\tlearn: 0.0020600\ttotal: 1.1s\tremaining: 22.6ms\n",
      "98:\tlearn: 0.0020304\ttotal: 1.12s\tremaining: 11.3ms\n",
      "99:\tlearn: 0.0019908\ttotal: 1.13s\tremaining: 0us\n",
      "0:\tlearn: 1.4127895\ttotal: 22.7ms\tremaining: 2.25s\n",
      "1:\tlearn: 1.1353385\ttotal: 43.7ms\tremaining: 2.14s\n",
      "2:\tlearn: 0.9439262\ttotal: 62.1ms\tremaining: 2.01s\n",
      "3:\tlearn: 0.8018176\ttotal: 78.7ms\tremaining: 1.89s\n",
      "4:\tlearn: 0.6913086\ttotal: 93.6ms\tremaining: 1.78s\n",
      "5:\tlearn: 0.5996302\ttotal: 109ms\tremaining: 1.7s\n",
      "6:\tlearn: 0.5263652\ttotal: 123ms\tremaining: 1.64s\n",
      "7:\tlearn: 0.4640471\ttotal: 137ms\tremaining: 1.58s\n",
      "8:\tlearn: 0.4117548\ttotal: 150ms\tremaining: 1.52s\n",
      "9:\tlearn: 0.3658873\ttotal: 163ms\tremaining: 1.47s\n",
      "10:\tlearn: 0.3264099\ttotal: 174ms\tremaining: 1.41s\n",
      "11:\tlearn: 0.2919711\ttotal: 184ms\tremaining: 1.35s\n",
      "12:\tlearn: 0.2625605\ttotal: 194ms\tremaining: 1.3s\n",
      "13:\tlearn: 0.2351082\ttotal: 205ms\tremaining: 1.26s\n",
      "14:\tlearn: 0.2110948\ttotal: 215ms\tremaining: 1.22s\n",
      "15:\tlearn: 0.1902327\ttotal: 225ms\tremaining: 1.18s\n",
      "16:\tlearn: 0.1716989\ttotal: 235ms\tremaining: 1.15s\n",
      "17:\tlearn: 0.1551737\ttotal: 245ms\tremaining: 1.12s\n",
      "18:\tlearn: 0.1410129\ttotal: 255ms\tremaining: 1.09s\n",
      "19:\tlearn: 0.1278266\ttotal: 265ms\tremaining: 1.06s\n",
      "20:\tlearn: 0.1162841\ttotal: 275ms\tremaining: 1.03s\n",
      "21:\tlearn: 0.1056849\ttotal: 284ms\tremaining: 1.01s\n",
      "22:\tlearn: 0.0961560\ttotal: 294ms\tremaining: 986ms\n",
      "23:\tlearn: 0.0877372\ttotal: 304ms\tremaining: 964ms\n",
      "24:\tlearn: 0.0798325\ttotal: 316ms\tremaining: 948ms\n",
      "25:\tlearn: 0.0727174\ttotal: 328ms\tremaining: 933ms\n",
      "26:\tlearn: 0.0662888\ttotal: 339ms\tremaining: 916ms\n",
      "27:\tlearn: 0.0602377\ttotal: 350ms\tremaining: 901ms\n",
      "28:\tlearn: 0.0548822\ttotal: 361ms\tremaining: 884ms\n",
      "29:\tlearn: 0.0500547\ttotal: 373ms\tremaining: 869ms\n",
      "30:\tlearn: 0.0461462\ttotal: 384ms\tremaining: 854ms\n",
      "31:\tlearn: 0.0421588\ttotal: 394ms\tremaining: 837ms\n",
      "32:\tlearn: 0.0386725\ttotal: 404ms\tremaining: 821ms\n",
      "33:\tlearn: 0.0354814\ttotal: 415ms\tremaining: 805ms\n",
      "34:\tlearn: 0.0326908\ttotal: 426ms\tremaining: 791ms\n",
      "35:\tlearn: 0.0302619\ttotal: 436ms\tremaining: 774ms\n",
      "36:\tlearn: 0.0279157\ttotal: 445ms\tremaining: 758ms\n",
      "37:\tlearn: 0.0254889\ttotal: 455ms\tremaining: 743ms\n",
      "38:\tlearn: 0.0233962\ttotal: 466ms\tremaining: 729ms\n",
      "39:\tlearn: 0.0216753\ttotal: 476ms\tremaining: 715ms\n",
      "40:\tlearn: 0.0199041\ttotal: 487ms\tremaining: 701ms\n",
      "41:\tlearn: 0.0185515\ttotal: 497ms\tremaining: 687ms\n",
      "42:\tlearn: 0.0171785\ttotal: 508ms\tremaining: 673ms\n",
      "43:\tlearn: 0.0159580\ttotal: 518ms\tremaining: 660ms\n",
      "44:\tlearn: 0.0149187\ttotal: 528ms\tremaining: 646ms\n",
      "45:\tlearn: 0.0139700\ttotal: 538ms\tremaining: 632ms\n",
      "46:\tlearn: 0.0129216\ttotal: 549ms\tremaining: 619ms\n",
      "47:\tlearn: 0.0120802\ttotal: 559ms\tremaining: 606ms\n",
      "48:\tlearn: 0.0114128\ttotal: 569ms\tremaining: 592ms\n",
      "49:\tlearn: 0.0107312\ttotal: 579ms\tremaining: 579ms\n",
      "50:\tlearn: 0.0099936\ttotal: 589ms\tremaining: 566ms\n",
      "51:\tlearn: 0.0094170\ttotal: 599ms\tremaining: 553ms\n",
      "52:\tlearn: 0.0088216\ttotal: 609ms\tremaining: 540ms\n",
      "53:\tlearn: 0.0083258\ttotal: 619ms\tremaining: 527ms\n",
      "54:\tlearn: 0.0079270\ttotal: 629ms\tremaining: 514ms\n",
      "55:\tlearn: 0.0074973\ttotal: 638ms\tremaining: 502ms\n",
      "56:\tlearn: 0.0071826\ttotal: 648ms\tremaining: 489ms\n",
      "57:\tlearn: 0.0068783\ttotal: 658ms\tremaining: 476ms\n",
      "58:\tlearn: 0.0065192\ttotal: 668ms\tremaining: 464ms\n",
      "59:\tlearn: 0.0062364\ttotal: 678ms\tremaining: 452ms\n",
      "60:\tlearn: 0.0059365\ttotal: 688ms\tremaining: 440ms\n",
      "61:\tlearn: 0.0056945\ttotal: 698ms\tremaining: 428ms\n",
      "62:\tlearn: 0.0054698\ttotal: 707ms\tremaining: 416ms\n",
      "63:\tlearn: 0.0052780\ttotal: 717ms\tremaining: 403ms\n",
      "64:\tlearn: 0.0050016\ttotal: 727ms\tremaining: 391ms\n",
      "65:\tlearn: 0.0047131\ttotal: 737ms\tremaining: 380ms\n",
      "66:\tlearn: 0.0045514\ttotal: 746ms\tremaining: 367ms\n",
      "67:\tlearn: 0.0044042\ttotal: 755ms\tremaining: 356ms\n",
      "68:\tlearn: 0.0042371\ttotal: 765ms\tremaining: 344ms\n",
      "69:\tlearn: 0.0041023\ttotal: 775ms\tremaining: 332ms\n",
      "70:\tlearn: 0.0038636\ttotal: 785ms\tremaining: 321ms\n",
      "71:\tlearn: 0.0037166\ttotal: 795ms\tremaining: 309ms\n",
      "72:\tlearn: 0.0035707\ttotal: 806ms\tremaining: 298ms\n",
      "73:\tlearn: 0.0034695\ttotal: 816ms\tremaining: 287ms\n",
      "74:\tlearn: 0.0033919\ttotal: 825ms\tremaining: 275ms\n",
      "75:\tlearn: 0.0032720\ttotal: 835ms\tremaining: 264ms\n",
      "76:\tlearn: 0.0031959\ttotal: 845ms\tremaining: 252ms\n",
      "77:\tlearn: 0.0030876\ttotal: 855ms\tremaining: 241ms\n",
      "78:\tlearn: 0.0029958\ttotal: 865ms\tremaining: 230ms\n",
      "79:\tlearn: 0.0029357\ttotal: 875ms\tremaining: 219ms\n",
      "80:\tlearn: 0.0028679\ttotal: 884ms\tremaining: 207ms\n",
      "81:\tlearn: 0.0027642\ttotal: 894ms\tremaining: 196ms\n",
      "82:\tlearn: 0.0027132\ttotal: 904ms\tremaining: 185ms\n",
      "83:\tlearn: 0.0026347\ttotal: 914ms\tremaining: 174ms\n",
      "84:\tlearn: 0.0025544\ttotal: 924ms\tremaining: 163ms\n",
      "85:\tlearn: 0.0024350\ttotal: 934ms\tremaining: 152ms\n",
      "86:\tlearn: 0.0023763\ttotal: 945ms\tremaining: 141ms\n",
      "87:\tlearn: 0.0023193\ttotal: 956ms\tremaining: 130ms\n",
      "88:\tlearn: 0.0022681\ttotal: 966ms\tremaining: 119ms\n",
      "89:\tlearn: 0.0022352\ttotal: 978ms\tremaining: 109ms\n",
      "90:\tlearn: 0.0021869\ttotal: 987ms\tremaining: 97.6ms\n",
      "91:\tlearn: 0.0021206\ttotal: 998ms\tremaining: 86.8ms\n",
      "92:\tlearn: 0.0020415\ttotal: 1.01s\tremaining: 76ms\n",
      "93:\tlearn: 0.0019808\ttotal: 1.02s\tremaining: 65.1ms\n",
      "94:\tlearn: 0.0019442\ttotal: 1.03s\tremaining: 54.3ms\n",
      "95:\tlearn: 0.0019023\ttotal: 1.04s\tremaining: 43.4ms\n",
      "96:\tlearn: 0.0018637\ttotal: 1.05s\tremaining: 32.5ms\n",
      "97:\tlearn: 0.0018183\ttotal: 1.06s\tremaining: 21.7ms\n",
      "98:\tlearn: 0.0017875\ttotal: 1.07s\tremaining: 10.8ms\n",
      "99:\tlearn: 0.0017443\ttotal: 1.08s\tremaining: 0us\n",
      "0:\tlearn: 1.4085940\ttotal: 23ms\tremaining: 2.28s\n",
      "1:\tlearn: 1.1326618\ttotal: 41ms\tremaining: 2.01s\n",
      "2:\tlearn: 0.9431160\ttotal: 56.6ms\tremaining: 1.83s\n",
      "3:\tlearn: 0.8017190\ttotal: 70.1ms\tremaining: 1.68s\n",
      "4:\tlearn: 0.6919850\ttotal: 82.9ms\tremaining: 1.57s\n",
      "5:\tlearn: 0.6005678\ttotal: 95.1ms\tremaining: 1.49s\n",
      "6:\tlearn: 0.5253034\ttotal: 106ms\tremaining: 1.41s\n",
      "7:\tlearn: 0.4642856\ttotal: 117ms\tremaining: 1.34s\n",
      "8:\tlearn: 0.4117191\ttotal: 127ms\tremaining: 1.28s\n",
      "9:\tlearn: 0.3659251\ttotal: 138ms\tremaining: 1.24s\n",
      "10:\tlearn: 0.3253223\ttotal: 148ms\tremaining: 1.2s\n",
      "11:\tlearn: 0.2910358\ttotal: 159ms\tremaining: 1.17s\n",
      "12:\tlearn: 0.2615681\ttotal: 171ms\tremaining: 1.14s\n",
      "13:\tlearn: 0.2354847\ttotal: 182ms\tremaining: 1.11s\n",
      "14:\tlearn: 0.2111817\ttotal: 192ms\tremaining: 1.09s\n",
      "15:\tlearn: 0.1903467\ttotal: 202ms\tremaining: 1.06s\n",
      "16:\tlearn: 0.1719001\ttotal: 212ms\tremaining: 1.04s\n",
      "17:\tlearn: 0.1549948\ttotal: 222ms\tremaining: 1.01s\n",
      "18:\tlearn: 0.1406719\ttotal: 233ms\tremaining: 993ms\n",
      "19:\tlearn: 0.1278609\ttotal: 243ms\tremaining: 972ms\n",
      "20:\tlearn: 0.1163128\ttotal: 253ms\tremaining: 953ms\n",
      "21:\tlearn: 0.1057801\ttotal: 263ms\tremaining: 934ms\n",
      "22:\tlearn: 0.0962538\ttotal: 274ms\tremaining: 916ms\n",
      "23:\tlearn: 0.0869784\ttotal: 283ms\tremaining: 897ms\n",
      "24:\tlearn: 0.0787199\ttotal: 293ms\tremaining: 880ms\n",
      "25:\tlearn: 0.0713836\ttotal: 305ms\tremaining: 867ms\n",
      "26:\tlearn: 0.0653052\ttotal: 314ms\tremaining: 849ms\n",
      "27:\tlearn: 0.0596375\ttotal: 324ms\tremaining: 833ms\n",
      "28:\tlearn: 0.0542217\ttotal: 334ms\tremaining: 817ms\n",
      "29:\tlearn: 0.0492108\ttotal: 344ms\tremaining: 803ms\n",
      "30:\tlearn: 0.0452540\ttotal: 357ms\tremaining: 794ms\n",
      "31:\tlearn: 0.0414021\ttotal: 367ms\tremaining: 780ms\n",
      "32:\tlearn: 0.0379286\ttotal: 377ms\tremaining: 765ms\n",
      "33:\tlearn: 0.0346586\ttotal: 387ms\tremaining: 751ms\n",
      "34:\tlearn: 0.0319382\ttotal: 396ms\tremaining: 735ms\n",
      "35:\tlearn: 0.0293539\ttotal: 405ms\tremaining: 721ms\n",
      "36:\tlearn: 0.0270939\ttotal: 416ms\tremaining: 707ms\n",
      "37:\tlearn: 0.0250947\ttotal: 425ms\tremaining: 694ms\n",
      "38:\tlearn: 0.0233086\ttotal: 435ms\tremaining: 681ms\n",
      "39:\tlearn: 0.0216757\ttotal: 445ms\tremaining: 668ms\n",
      "40:\tlearn: 0.0202452\ttotal: 455ms\tremaining: 654ms\n",
      "41:\tlearn: 0.0189179\ttotal: 465ms\tremaining: 642ms\n",
      "42:\tlearn: 0.0175213\ttotal: 474ms\tremaining: 629ms\n",
      "43:\tlearn: 0.0164616\ttotal: 484ms\tremaining: 615ms\n",
      "44:\tlearn: 0.0154618\ttotal: 493ms\tremaining: 603ms\n",
      "45:\tlearn: 0.0145043\ttotal: 502ms\tremaining: 590ms\n",
      "46:\tlearn: 0.0137924\ttotal: 512ms\tremaining: 578ms\n",
      "47:\tlearn: 0.0127941\ttotal: 523ms\tremaining: 566ms\n",
      "48:\tlearn: 0.0121007\ttotal: 533ms\tremaining: 555ms\n",
      "49:\tlearn: 0.0114974\ttotal: 543ms\tremaining: 543ms\n",
      "50:\tlearn: 0.0106676\ttotal: 554ms\tremaining: 533ms\n",
      "51:\tlearn: 0.0099779\ttotal: 564ms\tremaining: 521ms\n",
      "52:\tlearn: 0.0094856\ttotal: 575ms\tremaining: 510ms\n",
      "53:\tlearn: 0.0090723\ttotal: 584ms\tremaining: 497ms\n",
      "54:\tlearn: 0.0085702\ttotal: 594ms\tremaining: 486ms\n",
      "55:\tlearn: 0.0080843\ttotal: 603ms\tremaining: 474ms\n",
      "56:\tlearn: 0.0076785\ttotal: 613ms\tremaining: 463ms\n",
      "57:\tlearn: 0.0072705\ttotal: 623ms\tremaining: 451ms\n",
      "58:\tlearn: 0.0069426\ttotal: 633ms\tremaining: 440ms\n",
      "59:\tlearn: 0.0066499\ttotal: 642ms\tremaining: 428ms\n",
      "60:\tlearn: 0.0063915\ttotal: 651ms\tremaining: 416ms\n",
      "61:\tlearn: 0.0060046\ttotal: 661ms\tremaining: 405ms\n",
      "62:\tlearn: 0.0057894\ttotal: 670ms\tremaining: 394ms\n",
      "63:\tlearn: 0.0056214\ttotal: 680ms\tremaining: 382ms\n",
      "64:\tlearn: 0.0053340\ttotal: 690ms\tremaining: 371ms\n",
      "65:\tlearn: 0.0051877\ttotal: 699ms\tremaining: 360ms\n",
      "66:\tlearn: 0.0049333\ttotal: 708ms\tremaining: 349ms\n",
      "67:\tlearn: 0.0048023\ttotal: 718ms\tremaining: 338ms\n",
      "68:\tlearn: 0.0045787\ttotal: 726ms\tremaining: 326ms\n",
      "69:\tlearn: 0.0044197\ttotal: 735ms\tremaining: 315ms\n",
      "70:\tlearn: 0.0043450\ttotal: 745ms\tremaining: 304ms\n",
      "71:\tlearn: 0.0041370\ttotal: 754ms\tremaining: 293ms\n",
      "72:\tlearn: 0.0039742\ttotal: 765ms\tremaining: 283ms\n",
      "73:\tlearn: 0.0038242\ttotal: 774ms\tremaining: 272ms\n",
      "74:\tlearn: 0.0037469\ttotal: 784ms\tremaining: 261ms\n",
      "75:\tlearn: 0.0036717\ttotal: 793ms\tremaining: 250ms\n",
      "76:\tlearn: 0.0035802\ttotal: 802ms\tremaining: 240ms\n",
      "77:\tlearn: 0.0035131\ttotal: 811ms\tremaining: 229ms\n",
      "78:\tlearn: 0.0033917\ttotal: 820ms\tremaining: 218ms\n",
      "79:\tlearn: 0.0032485\ttotal: 830ms\tremaining: 208ms\n",
      "80:\tlearn: 0.0031254\ttotal: 840ms\tremaining: 197ms\n",
      "81:\tlearn: 0.0029725\ttotal: 850ms\tremaining: 187ms\n",
      "82:\tlearn: 0.0029106\ttotal: 858ms\tremaining: 176ms\n",
      "83:\tlearn: 0.0028256\ttotal: 868ms\tremaining: 165ms\n",
      "84:\tlearn: 0.0027567\ttotal: 879ms\tremaining: 155ms\n",
      "85:\tlearn: 0.0027328\ttotal: 888ms\tremaining: 145ms\n",
      "86:\tlearn: 0.0025774\ttotal: 899ms\tremaining: 134ms\n",
      "87:\tlearn: 0.0025012\ttotal: 909ms\tremaining: 124ms\n",
      "88:\tlearn: 0.0024683\ttotal: 919ms\tremaining: 114ms\n",
      "89:\tlearn: 0.0023777\ttotal: 929ms\tremaining: 103ms\n",
      "90:\tlearn: 0.0022774\ttotal: 938ms\tremaining: 92.8ms\n",
      "91:\tlearn: 0.0022472\ttotal: 947ms\tremaining: 82.3ms\n",
      "92:\tlearn: 0.0021779\ttotal: 956ms\tremaining: 71.9ms\n",
      "93:\tlearn: 0.0020769\ttotal: 966ms\tremaining: 61.7ms\n",
      "94:\tlearn: 0.0020394\ttotal: 975ms\tremaining: 51.3ms\n",
      "95:\tlearn: 0.0019900\ttotal: 985ms\tremaining: 41ms\n",
      "96:\tlearn: 0.0019053\ttotal: 995ms\tremaining: 30.8ms\n",
      "97:\tlearn: 0.0018347\ttotal: 1s\tremaining: 20.5ms\n",
      "98:\tlearn: 0.0017871\ttotal: 1.01s\tremaining: 10.2ms\n",
      "99:\tlearn: 0.0017687\ttotal: 1.02s\tremaining: 0us\n",
      "0:\tlearn: 1.4108397\ttotal: 23.9ms\tremaining: 2.37s\n",
      "1:\tlearn: 1.1326837\ttotal: 45.1ms\tremaining: 2.21s\n",
      "2:\tlearn: 0.9402512\ttotal: 64.6ms\tremaining: 2.09s\n",
      "3:\tlearn: 0.7999597\ttotal: 81.1ms\tremaining: 1.95s\n",
      "4:\tlearn: 0.6892595\ttotal: 97.1ms\tremaining: 1.84s\n",
      "5:\tlearn: 0.5985880\ttotal: 112ms\tremaining: 1.75s\n",
      "6:\tlearn: 0.5259767\ttotal: 127ms\tremaining: 1.69s\n",
      "7:\tlearn: 0.4631179\ttotal: 141ms\tremaining: 1.62s\n",
      "8:\tlearn: 0.4108513\ttotal: 154ms\tremaining: 1.55s\n",
      "9:\tlearn: 0.3647378\ttotal: 168ms\tremaining: 1.51s\n",
      "10:\tlearn: 0.3255982\ttotal: 181ms\tremaining: 1.46s\n",
      "11:\tlearn: 0.2910757\ttotal: 192ms\tremaining: 1.41s\n",
      "12:\tlearn: 0.2610123\ttotal: 204ms\tremaining: 1.37s\n",
      "13:\tlearn: 0.2345283\ttotal: 216ms\tremaining: 1.32s\n",
      "14:\tlearn: 0.2103357\ttotal: 227ms\tremaining: 1.29s\n",
      "15:\tlearn: 0.1896556\ttotal: 238ms\tremaining: 1.25s\n",
      "16:\tlearn: 0.1710218\ttotal: 248ms\tremaining: 1.21s\n",
      "17:\tlearn: 0.1544805\ttotal: 258ms\tremaining: 1.18s\n",
      "18:\tlearn: 0.1401730\ttotal: 268ms\tremaining: 1.14s\n",
      "19:\tlearn: 0.1268044\ttotal: 278ms\tremaining: 1.11s\n",
      "20:\tlearn: 0.1152712\ttotal: 287ms\tremaining: 1.08s\n",
      "21:\tlearn: 0.1044173\ttotal: 297ms\tremaining: 1.05s\n",
      "22:\tlearn: 0.0948748\ttotal: 307ms\tremaining: 1.03s\n",
      "23:\tlearn: 0.0866081\ttotal: 317ms\tremaining: 1s\n",
      "24:\tlearn: 0.0789520\ttotal: 327ms\tremaining: 981ms\n",
      "25:\tlearn: 0.0714636\ttotal: 337ms\tremaining: 959ms\n",
      "26:\tlearn: 0.0652662\ttotal: 346ms\tremaining: 937ms\n",
      "27:\tlearn: 0.0596685\ttotal: 356ms\tremaining: 916ms\n",
      "28:\tlearn: 0.0544743\ttotal: 366ms\tremaining: 896ms\n",
      "29:\tlearn: 0.0494365\ttotal: 375ms\tremaining: 876ms\n",
      "30:\tlearn: 0.0454416\ttotal: 385ms\tremaining: 856ms\n",
      "31:\tlearn: 0.0414652\ttotal: 396ms\tremaining: 841ms\n",
      "32:\tlearn: 0.0378357\ttotal: 406ms\tremaining: 825ms\n",
      "33:\tlearn: 0.0348307\ttotal: 416ms\tremaining: 808ms\n",
      "34:\tlearn: 0.0319447\ttotal: 426ms\tremaining: 791ms\n",
      "35:\tlearn: 0.0292336\ttotal: 437ms\tremaining: 776ms\n",
      "36:\tlearn: 0.0269578\ttotal: 447ms\tremaining: 761ms\n",
      "37:\tlearn: 0.0249161\ttotal: 457ms\tremaining: 746ms\n",
      "38:\tlearn: 0.0228673\ttotal: 467ms\tremaining: 731ms\n",
      "39:\tlearn: 0.0211037\ttotal: 477ms\tremaining: 716ms\n",
      "40:\tlearn: 0.0195293\ttotal: 487ms\tremaining: 701ms\n",
      "41:\tlearn: 0.0182712\ttotal: 497ms\tremaining: 686ms\n",
      "42:\tlearn: 0.0170932\ttotal: 506ms\tremaining: 670ms\n",
      "43:\tlearn: 0.0157608\ttotal: 515ms\tremaining: 656ms\n",
      "44:\tlearn: 0.0148312\ttotal: 524ms\tremaining: 640ms\n",
      "45:\tlearn: 0.0139620\ttotal: 533ms\tremaining: 626ms\n",
      "46:\tlearn: 0.0130963\ttotal: 542ms\tremaining: 612ms\n",
      "47:\tlearn: 0.0123070\ttotal: 551ms\tremaining: 597ms\n",
      "48:\tlearn: 0.0114789\ttotal: 561ms\tremaining: 584ms\n",
      "49:\tlearn: 0.0109141\ttotal: 570ms\tremaining: 570ms\n",
      "50:\tlearn: 0.0102356\ttotal: 580ms\tremaining: 557ms\n",
      "51:\tlearn: 0.0096994\ttotal: 589ms\tremaining: 544ms\n",
      "52:\tlearn: 0.0091417\ttotal: 598ms\tremaining: 530ms\n",
      "53:\tlearn: 0.0087052\ttotal: 607ms\tremaining: 517ms\n",
      "54:\tlearn: 0.0082618\ttotal: 616ms\tremaining: 504ms\n",
      "55:\tlearn: 0.0078314\ttotal: 626ms\tremaining: 492ms\n",
      "56:\tlearn: 0.0073149\ttotal: 636ms\tremaining: 480ms\n",
      "57:\tlearn: 0.0069766\ttotal: 646ms\tremaining: 468ms\n",
      "58:\tlearn: 0.0066456\ttotal: 656ms\tremaining: 456ms\n",
      "59:\tlearn: 0.0062386\ttotal: 666ms\tremaining: 444ms\n",
      "60:\tlearn: 0.0058292\ttotal: 676ms\tremaining: 432ms\n",
      "61:\tlearn: 0.0056114\ttotal: 685ms\tremaining: 420ms\n",
      "62:\tlearn: 0.0054346\ttotal: 695ms\tremaining: 408ms\n",
      "63:\tlearn: 0.0052504\ttotal: 705ms\tremaining: 397ms\n",
      "64:\tlearn: 0.0049273\ttotal: 716ms\tremaining: 385ms\n",
      "65:\tlearn: 0.0046801\ttotal: 726ms\tremaining: 374ms\n",
      "66:\tlearn: 0.0044717\ttotal: 735ms\tremaining: 362ms\n",
      "67:\tlearn: 0.0043084\ttotal: 744ms\tremaining: 350ms\n",
      "68:\tlearn: 0.0041119\ttotal: 753ms\tremaining: 338ms\n",
      "69:\tlearn: 0.0039217\ttotal: 763ms\tremaining: 327ms\n",
      "70:\tlearn: 0.0038166\ttotal: 771ms\tremaining: 315ms\n",
      "71:\tlearn: 0.0037023\ttotal: 780ms\tremaining: 303ms\n",
      "72:\tlearn: 0.0035907\ttotal: 790ms\tremaining: 292ms\n",
      "73:\tlearn: 0.0034363\ttotal: 799ms\tremaining: 281ms\n",
      "74:\tlearn: 0.0033492\ttotal: 808ms\tremaining: 269ms\n",
      "75:\tlearn: 0.0032394\ttotal: 817ms\tremaining: 258ms\n",
      "76:\tlearn: 0.0031394\ttotal: 826ms\tremaining: 247ms\n",
      "77:\tlearn: 0.0030421\ttotal: 835ms\tremaining: 235ms\n",
      "78:\tlearn: 0.0029666\ttotal: 844ms\tremaining: 224ms\n",
      "79:\tlearn: 0.0029131\ttotal: 853ms\tremaining: 213ms\n",
      "80:\tlearn: 0.0027767\ttotal: 863ms\tremaining: 202ms\n",
      "81:\tlearn: 0.0027111\ttotal: 872ms\tremaining: 191ms\n",
      "82:\tlearn: 0.0026356\ttotal: 880ms\tremaining: 180ms\n",
      "83:\tlearn: 0.0025651\ttotal: 890ms\tremaining: 169ms\n",
      "84:\tlearn: 0.0025142\ttotal: 899ms\tremaining: 159ms\n",
      "85:\tlearn: 0.0024416\ttotal: 910ms\tremaining: 148ms\n",
      "86:\tlearn: 0.0023299\ttotal: 921ms\tremaining: 138ms\n",
      "87:\tlearn: 0.0022707\ttotal: 932ms\tremaining: 127ms\n",
      "88:\tlearn: 0.0022168\ttotal: 941ms\tremaining: 116ms\n",
      "89:\tlearn: 0.0021847\ttotal: 950ms\tremaining: 106ms\n",
      "90:\tlearn: 0.0021229\ttotal: 961ms\tremaining: 95.1ms\n",
      "91:\tlearn: 0.0020794\ttotal: 971ms\tremaining: 84.5ms\n",
      "92:\tlearn: 0.0020386\ttotal: 982ms\tremaining: 73.9ms\n",
      "93:\tlearn: 0.0019934\ttotal: 992ms\tremaining: 63.3ms\n",
      "94:\tlearn: 0.0019634\ttotal: 1s\tremaining: 52.7ms\n",
      "95:\tlearn: 0.0019037\ttotal: 1.01s\tremaining: 42.1ms\n",
      "96:\tlearn: 0.0018681\ttotal: 1.02s\tremaining: 31.5ms\n",
      "97:\tlearn: 0.0017833\ttotal: 1.03s\tremaining: 21ms\n",
      "98:\tlearn: 0.0017590\ttotal: 1.04s\tremaining: 10.5ms\n",
      "99:\tlearn: 0.0016760\ttotal: 1.05s\tremaining: 0us\n",
      "0:\tlearn: 1.4083245\ttotal: 20.1ms\tremaining: 1.99s\n",
      "1:\tlearn: 1.1324455\ttotal: 38.9ms\tremaining: 1.91s\n",
      "2:\tlearn: 0.9409426\ttotal: 55.9ms\tremaining: 1.81s\n",
      "3:\tlearn: 0.8006845\ttotal: 72.2ms\tremaining: 1.73s\n",
      "4:\tlearn: 0.6914052\ttotal: 86.5ms\tremaining: 1.64s\n",
      "5:\tlearn: 0.6004181\ttotal: 100ms\tremaining: 1.57s\n",
      "6:\tlearn: 0.5254043\ttotal: 113ms\tremaining: 1.5s\n",
      "7:\tlearn: 0.4612714\ttotal: 125ms\tremaining: 1.43s\n",
      "8:\tlearn: 0.4090473\ttotal: 136ms\tremaining: 1.38s\n",
      "9:\tlearn: 0.3635490\ttotal: 148ms\tremaining: 1.33s\n",
      "10:\tlearn: 0.3233676\ttotal: 159ms\tremaining: 1.29s\n",
      "11:\tlearn: 0.2892096\ttotal: 170ms\tremaining: 1.25s\n",
      "12:\tlearn: 0.2597620\ttotal: 180ms\tremaining: 1.21s\n",
      "13:\tlearn: 0.2338851\ttotal: 190ms\tremaining: 1.17s\n",
      "14:\tlearn: 0.2110764\ttotal: 200ms\tremaining: 1.13s\n",
      "15:\tlearn: 0.1903899\ttotal: 209ms\tremaining: 1.1s\n",
      "16:\tlearn: 0.1719536\ttotal: 219ms\tremaining: 1.07s\n",
      "17:\tlearn: 0.1554822\ttotal: 228ms\tremaining: 1.04s\n",
      "18:\tlearn: 0.1412965\ttotal: 238ms\tremaining: 1.01s\n",
      "19:\tlearn: 0.1277681\ttotal: 247ms\tremaining: 989ms\n",
      "20:\tlearn: 0.1158967\ttotal: 257ms\tremaining: 965ms\n",
      "21:\tlearn: 0.1054315\ttotal: 267ms\tremaining: 946ms\n",
      "22:\tlearn: 0.0956928\ttotal: 277ms\tremaining: 927ms\n",
      "23:\tlearn: 0.0872002\ttotal: 287ms\tremaining: 907ms\n",
      "24:\tlearn: 0.0791619\ttotal: 298ms\tremaining: 893ms\n",
      "25:\tlearn: 0.0717231\ttotal: 308ms\tremaining: 877ms\n",
      "26:\tlearn: 0.0656283\ttotal: 318ms\tremaining: 859ms\n",
      "27:\tlearn: 0.0598809\ttotal: 328ms\tremaining: 843ms\n",
      "28:\tlearn: 0.0544079\ttotal: 338ms\tremaining: 827ms\n",
      "29:\tlearn: 0.0496873\ttotal: 349ms\tremaining: 814ms\n",
      "30:\tlearn: 0.0455648\ttotal: 359ms\tremaining: 799ms\n",
      "31:\tlearn: 0.0418372\ttotal: 369ms\tremaining: 784ms\n",
      "32:\tlearn: 0.0384428\ttotal: 379ms\tremaining: 769ms\n",
      "33:\tlearn: 0.0353483\ttotal: 388ms\tremaining: 754ms\n",
      "34:\tlearn: 0.0325734\ttotal: 398ms\tremaining: 739ms\n",
      "35:\tlearn: 0.0300175\ttotal: 407ms\tremaining: 724ms\n",
      "36:\tlearn: 0.0276629\ttotal: 417ms\tremaining: 711ms\n",
      "37:\tlearn: 0.0255865\ttotal: 427ms\tremaining: 697ms\n",
      "38:\tlearn: 0.0236196\ttotal: 437ms\tremaining: 683ms\n",
      "39:\tlearn: 0.0221166\ttotal: 446ms\tremaining: 669ms\n",
      "40:\tlearn: 0.0205744\ttotal: 455ms\tremaining: 655ms\n",
      "41:\tlearn: 0.0190043\ttotal: 465ms\tremaining: 642ms\n",
      "42:\tlearn: 0.0177953\ttotal: 474ms\tremaining: 628ms\n",
      "43:\tlearn: 0.0165839\ttotal: 483ms\tremaining: 614ms\n",
      "44:\tlearn: 0.0158242\ttotal: 490ms\tremaining: 598ms\n",
      "45:\tlearn: 0.0146561\ttotal: 499ms\tremaining: 586ms\n",
      "46:\tlearn: 0.0137302\ttotal: 508ms\tremaining: 573ms\n",
      "47:\tlearn: 0.0129135\ttotal: 518ms\tremaining: 561ms\n",
      "48:\tlearn: 0.0121496\ttotal: 527ms\tremaining: 549ms\n",
      "49:\tlearn: 0.0114504\ttotal: 536ms\tremaining: 536ms\n",
      "50:\tlearn: 0.0106816\ttotal: 547ms\tremaining: 525ms\n",
      "51:\tlearn: 0.0101383\ttotal: 557ms\tremaining: 514ms\n",
      "52:\tlearn: 0.0096446\ttotal: 567ms\tremaining: 503ms\n",
      "53:\tlearn: 0.0091732\ttotal: 577ms\tremaining: 491ms\n",
      "54:\tlearn: 0.0088213\ttotal: 587ms\tremaining: 480ms\n",
      "55:\tlearn: 0.0083982\ttotal: 597ms\tremaining: 469ms\n",
      "56:\tlearn: 0.0079736\ttotal: 607ms\tremaining: 458ms\n",
      "57:\tlearn: 0.0076927\ttotal: 617ms\tremaining: 447ms\n",
      "58:\tlearn: 0.0074473\ttotal: 626ms\tremaining: 435ms\n",
      "59:\tlearn: 0.0071304\ttotal: 635ms\tremaining: 424ms\n",
      "60:\tlearn: 0.0068154\ttotal: 645ms\tremaining: 412ms\n",
      "61:\tlearn: 0.0065636\ttotal: 654ms\tremaining: 401ms\n",
      "62:\tlearn: 0.0063628\ttotal: 664ms\tremaining: 390ms\n",
      "63:\tlearn: 0.0062071\ttotal: 674ms\tremaining: 379ms\n",
      "64:\tlearn: 0.0059994\ttotal: 683ms\tremaining: 368ms\n",
      "65:\tlearn: 0.0057454\ttotal: 693ms\tremaining: 357ms\n",
      "66:\tlearn: 0.0054128\ttotal: 703ms\tremaining: 346ms\n",
      "67:\tlearn: 0.0052573\ttotal: 712ms\tremaining: 335ms\n",
      "68:\tlearn: 0.0050701\ttotal: 722ms\tremaining: 324ms\n",
      "69:\tlearn: 0.0048675\ttotal: 731ms\tremaining: 313ms\n",
      "70:\tlearn: 0.0046308\ttotal: 741ms\tremaining: 303ms\n",
      "71:\tlearn: 0.0045371\ttotal: 749ms\tremaining: 291ms\n",
      "72:\tlearn: 0.0042768\ttotal: 760ms\tremaining: 281ms\n",
      "73:\tlearn: 0.0041815\ttotal: 769ms\tremaining: 270ms\n",
      "74:\tlearn: 0.0039525\ttotal: 778ms\tremaining: 259ms\n",
      "75:\tlearn: 0.0038652\ttotal: 788ms\tremaining: 249ms\n",
      "76:\tlearn: 0.0036094\ttotal: 798ms\tremaining: 238ms\n",
      "77:\tlearn: 0.0034472\ttotal: 809ms\tremaining: 228ms\n",
      "78:\tlearn: 0.0033818\ttotal: 819ms\tremaining: 218ms\n",
      "79:\tlearn: 0.0033180\ttotal: 828ms\tremaining: 207ms\n",
      "80:\tlearn: 0.0032421\ttotal: 839ms\tremaining: 197ms\n",
      "81:\tlearn: 0.0031831\ttotal: 850ms\tremaining: 187ms\n",
      "82:\tlearn: 0.0030446\ttotal: 863ms\tremaining: 177ms\n",
      "83:\tlearn: 0.0029916\ttotal: 874ms\tremaining: 166ms\n",
      "84:\tlearn: 0.0028454\ttotal: 885ms\tremaining: 156ms\n",
      "85:\tlearn: 0.0027423\ttotal: 896ms\tremaining: 146ms\n",
      "86:\tlearn: 0.0026841\ttotal: 906ms\tremaining: 135ms\n",
      "87:\tlearn: 0.0026521\ttotal: 913ms\tremaining: 125ms\n",
      "88:\tlearn: 0.0025930\ttotal: 924ms\tremaining: 114ms\n",
      "89:\tlearn: 0.0025517\ttotal: 935ms\tremaining: 104ms\n",
      "90:\tlearn: 0.0025024\ttotal: 945ms\tremaining: 93.5ms\n",
      "91:\tlearn: 0.0024480\ttotal: 955ms\tremaining: 83ms\n",
      "92:\tlearn: 0.0023485\ttotal: 966ms\tremaining: 72.7ms\n",
      "93:\tlearn: 0.0022600\ttotal: 976ms\tremaining: 62.3ms\n",
      "94:\tlearn: 0.0022319\ttotal: 986ms\tremaining: 51.9ms\n",
      "95:\tlearn: 0.0021886\ttotal: 997ms\tremaining: 41.5ms\n",
      "96:\tlearn: 0.0021343\ttotal: 1s\tremaining: 31.1ms\n",
      "97:\tlearn: 0.0021023\ttotal: 1.02s\tremaining: 20.7ms\n",
      "98:\tlearn: 0.0020747\ttotal: 1.03s\tremaining: 10.4ms\n",
      "99:\tlearn: 0.0020715\ttotal: 1.03s\tremaining: 0us\n",
      "0:\tlearn: 1.4072901\ttotal: 22.9ms\tremaining: 2.27s\n",
      "1:\tlearn: 1.1309453\ttotal: 41.9ms\tremaining: 2.06s\n",
      "2:\tlearn: 0.9413700\ttotal: 61.1ms\tremaining: 1.98s\n",
      "3:\tlearn: 0.8005549\ttotal: 80.1ms\tremaining: 1.92s\n",
      "4:\tlearn: 0.6906755\ttotal: 95.8ms\tremaining: 1.82s\n",
      "5:\tlearn: 0.6016846\ttotal: 111ms\tremaining: 1.74s\n",
      "6:\tlearn: 0.5260585\ttotal: 125ms\tremaining: 1.66s\n",
      "7:\tlearn: 0.4616762\ttotal: 139ms\tremaining: 1.59s\n",
      "8:\tlearn: 0.4092878\ttotal: 152ms\tremaining: 1.53s\n",
      "9:\tlearn: 0.3641962\ttotal: 164ms\tremaining: 1.48s\n",
      "10:\tlearn: 0.3239927\ttotal: 175ms\tremaining: 1.42s\n",
      "11:\tlearn: 0.2897119\ttotal: 187ms\tremaining: 1.37s\n",
      "12:\tlearn: 0.2594353\ttotal: 198ms\tremaining: 1.33s\n",
      "13:\tlearn: 0.2330866\ttotal: 209ms\tremaining: 1.28s\n",
      "14:\tlearn: 0.2099078\ttotal: 221ms\tremaining: 1.25s\n",
      "15:\tlearn: 0.1891062\ttotal: 232ms\tremaining: 1.22s\n",
      "16:\tlearn: 0.1704456\ttotal: 242ms\tremaining: 1.18s\n",
      "17:\tlearn: 0.1541735\ttotal: 252ms\tremaining: 1.15s\n",
      "18:\tlearn: 0.1398809\ttotal: 263ms\tremaining: 1.12s\n",
      "19:\tlearn: 0.1265491\ttotal: 274ms\tremaining: 1.1s\n",
      "20:\tlearn: 0.1147042\ttotal: 285ms\tremaining: 1.07s\n",
      "21:\tlearn: 0.1040769\ttotal: 294ms\tremaining: 1.04s\n",
      "22:\tlearn: 0.0946514\ttotal: 305ms\tremaining: 1.02s\n",
      "23:\tlearn: 0.0862523\ttotal: 316ms\tremaining: 1s\n",
      "24:\tlearn: 0.0783550\ttotal: 327ms\tremaining: 981ms\n",
      "25:\tlearn: 0.0710007\ttotal: 337ms\tremaining: 960ms\n",
      "26:\tlearn: 0.0648944\ttotal: 347ms\tremaining: 939ms\n",
      "27:\tlearn: 0.0593249\ttotal: 358ms\tremaining: 920ms\n",
      "28:\tlearn: 0.0539082\ttotal: 368ms\tremaining: 902ms\n",
      "29:\tlearn: 0.0493052\ttotal: 379ms\tremaining: 885ms\n",
      "30:\tlearn: 0.0453204\ttotal: 390ms\tremaining: 869ms\n",
      "31:\tlearn: 0.0413831\ttotal: 402ms\tremaining: 854ms\n",
      "32:\tlearn: 0.0379151\ttotal: 413ms\tremaining: 838ms\n",
      "33:\tlearn: 0.0347190\ttotal: 425ms\tremaining: 825ms\n",
      "34:\tlearn: 0.0318176\ttotal: 436ms\tremaining: 809ms\n",
      "35:\tlearn: 0.0293731\ttotal: 447ms\tremaining: 795ms\n",
      "36:\tlearn: 0.0271326\ttotal: 458ms\tremaining: 780ms\n",
      "37:\tlearn: 0.0248979\ttotal: 470ms\tremaining: 767ms\n",
      "38:\tlearn: 0.0228516\ttotal: 481ms\tremaining: 752ms\n",
      "39:\tlearn: 0.0211055\ttotal: 491ms\tremaining: 737ms\n",
      "40:\tlearn: 0.0196883\ttotal: 502ms\tremaining: 722ms\n",
      "41:\tlearn: 0.0182000\ttotal: 512ms\tremaining: 707ms\n",
      "42:\tlearn: 0.0169521\ttotal: 522ms\tremaining: 691ms\n",
      "43:\tlearn: 0.0157728\ttotal: 531ms\tremaining: 676ms\n",
      "44:\tlearn: 0.0147803\ttotal: 541ms\tremaining: 661ms\n",
      "45:\tlearn: 0.0139011\ttotal: 551ms\tremaining: 647ms\n",
      "46:\tlearn: 0.0131375\ttotal: 561ms\tremaining: 632ms\n",
      "47:\tlearn: 0.0123557\ttotal: 571ms\tremaining: 618ms\n",
      "48:\tlearn: 0.0116339\ttotal: 580ms\tremaining: 604ms\n",
      "49:\tlearn: 0.0109926\ttotal: 590ms\tremaining: 590ms\n",
      "50:\tlearn: 0.0102284\ttotal: 601ms\tremaining: 577ms\n",
      "51:\tlearn: 0.0095353\ttotal: 611ms\tremaining: 564ms\n",
      "52:\tlearn: 0.0090523\ttotal: 621ms\tremaining: 551ms\n",
      "53:\tlearn: 0.0085838\ttotal: 631ms\tremaining: 538ms\n",
      "54:\tlearn: 0.0081647\ttotal: 641ms\tremaining: 525ms\n",
      "55:\tlearn: 0.0078266\ttotal: 651ms\tremaining: 512ms\n",
      "56:\tlearn: 0.0074188\ttotal: 662ms\tremaining: 500ms\n",
      "57:\tlearn: 0.0070837\ttotal: 673ms\tremaining: 487ms\n",
      "58:\tlearn: 0.0067583\ttotal: 684ms\tremaining: 475ms\n",
      "59:\tlearn: 0.0064185\ttotal: 696ms\tremaining: 464ms\n",
      "60:\tlearn: 0.0061953\ttotal: 705ms\tremaining: 451ms\n",
      "61:\tlearn: 0.0059426\ttotal: 716ms\tremaining: 439ms\n",
      "62:\tlearn: 0.0056064\ttotal: 727ms\tremaining: 427ms\n",
      "63:\tlearn: 0.0054079\ttotal: 738ms\tremaining: 415ms\n",
      "64:\tlearn: 0.0052094\ttotal: 749ms\tremaining: 403ms\n",
      "65:\tlearn: 0.0050639\ttotal: 759ms\tremaining: 391ms\n",
      "66:\tlearn: 0.0048395\ttotal: 770ms\tremaining: 379ms\n",
      "67:\tlearn: 0.0046809\ttotal: 780ms\tremaining: 367ms\n",
      "68:\tlearn: 0.0044648\ttotal: 800ms\tremaining: 359ms\n",
      "69:\tlearn: 0.0042841\ttotal: 810ms\tremaining: 347ms\n",
      "70:\tlearn: 0.0040811\ttotal: 820ms\tremaining: 335ms\n",
      "71:\tlearn: 0.0039464\ttotal: 829ms\tremaining: 322ms\n",
      "72:\tlearn: 0.0037353\ttotal: 839ms\tremaining: 310ms\n",
      "73:\tlearn: 0.0036254\ttotal: 849ms\tremaining: 298ms\n",
      "74:\tlearn: 0.0035270\ttotal: 859ms\tremaining: 286ms\n",
      "75:\tlearn: 0.0034560\ttotal: 868ms\tremaining: 274ms\n",
      "76:\tlearn: 0.0033323\ttotal: 877ms\tremaining: 262ms\n",
      "77:\tlearn: 0.0032438\ttotal: 887ms\tremaining: 250ms\n",
      "78:\tlearn: 0.0031434\ttotal: 897ms\tremaining: 238ms\n",
      "79:\tlearn: 0.0030740\ttotal: 907ms\tremaining: 227ms\n",
      "80:\tlearn: 0.0030103\ttotal: 916ms\tremaining: 215ms\n",
      "81:\tlearn: 0.0029194\ttotal: 926ms\tremaining: 203ms\n",
      "82:\tlearn: 0.0028813\ttotal: 935ms\tremaining: 192ms\n",
      "83:\tlearn: 0.0028271\ttotal: 945ms\tremaining: 180ms\n",
      "84:\tlearn: 0.0026937\ttotal: 957ms\tremaining: 169ms\n",
      "85:\tlearn: 0.0025884\ttotal: 968ms\tremaining: 158ms\n",
      "86:\tlearn: 0.0025345\ttotal: 979ms\tremaining: 146ms\n",
      "87:\tlearn: 0.0024662\ttotal: 990ms\tremaining: 135ms\n",
      "88:\tlearn: 0.0024146\ttotal: 1000ms\tremaining: 124ms\n",
      "89:\tlearn: 0.0023845\ttotal: 1.01s\tremaining: 112ms\n",
      "90:\tlearn: 0.0023232\ttotal: 1.02s\tremaining: 101ms\n",
      "91:\tlearn: 0.0023056\ttotal: 1.03s\tremaining: 89.4ms\n",
      "92:\tlearn: 0.0022807\ttotal: 1.04s\tremaining: 78.1ms\n",
      "93:\tlearn: 0.0022417\ttotal: 1.05s\tremaining: 66.8ms\n",
      "94:\tlearn: 0.0021989\ttotal: 1.06s\tremaining: 55.6ms\n",
      "95:\tlearn: 0.0021553\ttotal: 1.06s\tremaining: 44.4ms\n",
      "96:\tlearn: 0.0020975\ttotal: 1.07s\tremaining: 33.3ms\n",
      "97:\tlearn: 0.0020024\ttotal: 1.08s\tremaining: 22.1ms\n",
      "98:\tlearn: 0.0019546\ttotal: 1.09s\tremaining: 11.1ms\n",
      "99:\tlearn: 0.0018679\ttotal: 1.1s\tremaining: 0us\n",
      "0:\tlearn: 1.4122770\ttotal: 23.6ms\tremaining: 2.34s\n",
      "1:\tlearn: 1.1348596\ttotal: 43.4ms\tremaining: 2.13s\n",
      "2:\tlearn: 0.9432529\ttotal: 60.4ms\tremaining: 1.95s\n",
      "3:\tlearn: 0.7999190\ttotal: 74.6ms\tremaining: 1.79s\n",
      "4:\tlearn: 0.6904367\ttotal: 88.6ms\tremaining: 1.68s\n",
      "5:\tlearn: 0.5994226\ttotal: 103ms\tremaining: 1.61s\n",
      "6:\tlearn: 0.5260911\ttotal: 116ms\tremaining: 1.54s\n",
      "7:\tlearn: 0.4646069\ttotal: 128ms\tremaining: 1.47s\n",
      "8:\tlearn: 0.4122171\ttotal: 140ms\tremaining: 1.42s\n",
      "9:\tlearn: 0.3659315\ttotal: 152ms\tremaining: 1.37s\n",
      "10:\tlearn: 0.3255595\ttotal: 163ms\tremaining: 1.31s\n",
      "11:\tlearn: 0.2911386\ttotal: 173ms\tremaining: 1.27s\n",
      "12:\tlearn: 0.2612292\ttotal: 184ms\tremaining: 1.23s\n",
      "13:\tlearn: 0.2338382\ttotal: 195ms\tremaining: 1.2s\n",
      "14:\tlearn: 0.2105845\ttotal: 205ms\tremaining: 1.16s\n",
      "15:\tlearn: 0.1897714\ttotal: 216ms\tremaining: 1.13s\n",
      "16:\tlearn: 0.1711585\ttotal: 225ms\tremaining: 1.1s\n",
      "17:\tlearn: 0.1548323\ttotal: 235ms\tremaining: 1.07s\n",
      "18:\tlearn: 0.1406494\ttotal: 245ms\tremaining: 1.04s\n",
      "19:\tlearn: 0.1273837\ttotal: 256ms\tremaining: 1.02s\n",
      "20:\tlearn: 0.1155851\ttotal: 266ms\tremaining: 1s\n",
      "21:\tlearn: 0.1050087\ttotal: 276ms\tremaining: 978ms\n",
      "22:\tlearn: 0.0954025\ttotal: 285ms\tremaining: 956ms\n",
      "23:\tlearn: 0.0870451\ttotal: 295ms\tremaining: 935ms\n",
      "24:\tlearn: 0.0789636\ttotal: 305ms\tremaining: 914ms\n",
      "25:\tlearn: 0.0716805\ttotal: 314ms\tremaining: 895ms\n",
      "26:\tlearn: 0.0655714\ttotal: 324ms\tremaining: 875ms\n",
      "27:\tlearn: 0.0599432\ttotal: 334ms\tremaining: 858ms\n",
      "28:\tlearn: 0.0547134\ttotal: 343ms\tremaining: 839ms\n",
      "29:\tlearn: 0.0496401\ttotal: 352ms\tremaining: 821ms\n",
      "30:\tlearn: 0.0454488\ttotal: 361ms\tremaining: 803ms\n",
      "31:\tlearn: 0.0416593\ttotal: 371ms\tremaining: 788ms\n",
      "32:\tlearn: 0.0383713\ttotal: 381ms\tremaining: 773ms\n",
      "33:\tlearn: 0.0354221\ttotal: 390ms\tremaining: 758ms\n",
      "34:\tlearn: 0.0327438\ttotal: 400ms\tremaining: 743ms\n",
      "35:\tlearn: 0.0300365\ttotal: 410ms\tremaining: 729ms\n",
      "36:\tlearn: 0.0277209\ttotal: 419ms\tremaining: 713ms\n",
      "37:\tlearn: 0.0256001\ttotal: 428ms\tremaining: 699ms\n",
      "38:\tlearn: 0.0236184\ttotal: 438ms\tremaining: 685ms\n",
      "39:\tlearn: 0.0218494\ttotal: 450ms\tremaining: 675ms\n",
      "40:\tlearn: 0.0202754\ttotal: 460ms\tremaining: 661ms\n",
      "41:\tlearn: 0.0189041\ttotal: 469ms\tremaining: 648ms\n",
      "42:\tlearn: 0.0175756\ttotal: 480ms\tremaining: 636ms\n",
      "43:\tlearn: 0.0163584\ttotal: 490ms\tremaining: 623ms\n",
      "44:\tlearn: 0.0153887\ttotal: 500ms\tremaining: 611ms\n",
      "45:\tlearn: 0.0146634\ttotal: 510ms\tremaining: 599ms\n",
      "46:\tlearn: 0.0138377\ttotal: 520ms\tremaining: 586ms\n",
      "47:\tlearn: 0.0129290\ttotal: 530ms\tremaining: 574ms\n",
      "48:\tlearn: 0.0119746\ttotal: 540ms\tremaining: 562ms\n",
      "49:\tlearn: 0.0112700\ttotal: 548ms\tremaining: 548ms\n",
      "50:\tlearn: 0.0106363\ttotal: 558ms\tremaining: 536ms\n",
      "51:\tlearn: 0.0100561\ttotal: 567ms\tremaining: 523ms\n",
      "52:\tlearn: 0.0094750\ttotal: 577ms\tremaining: 512ms\n",
      "53:\tlearn: 0.0089967\ttotal: 586ms\tremaining: 499ms\n",
      "54:\tlearn: 0.0085560\ttotal: 596ms\tremaining: 488ms\n",
      "55:\tlearn: 0.0081134\ttotal: 606ms\tremaining: 476ms\n",
      "56:\tlearn: 0.0076465\ttotal: 616ms\tremaining: 465ms\n",
      "57:\tlearn: 0.0072855\ttotal: 625ms\tremaining: 453ms\n",
      "58:\tlearn: 0.0069646\ttotal: 635ms\tremaining: 441ms\n",
      "59:\tlearn: 0.0067050\ttotal: 644ms\tremaining: 429ms\n",
      "60:\tlearn: 0.0064955\ttotal: 652ms\tremaining: 417ms\n",
      "61:\tlearn: 0.0062448\ttotal: 661ms\tremaining: 405ms\n",
      "62:\tlearn: 0.0060135\ttotal: 670ms\tremaining: 394ms\n",
      "63:\tlearn: 0.0057957\ttotal: 679ms\tremaining: 382ms\n",
      "64:\tlearn: 0.0054744\ttotal: 691ms\tremaining: 372ms\n",
      "65:\tlearn: 0.0052457\ttotal: 701ms\tremaining: 361ms\n",
      "66:\tlearn: 0.0050513\ttotal: 711ms\tremaining: 350ms\n",
      "67:\tlearn: 0.0049060\ttotal: 720ms\tremaining: 339ms\n",
      "68:\tlearn: 0.0047606\ttotal: 730ms\tremaining: 328ms\n",
      "69:\tlearn: 0.0045452\ttotal: 740ms\tremaining: 317ms\n",
      "70:\tlearn: 0.0043577\ttotal: 751ms\tremaining: 307ms\n",
      "71:\tlearn: 0.0042284\ttotal: 760ms\tremaining: 296ms\n",
      "72:\tlearn: 0.0041019\ttotal: 770ms\tremaining: 285ms\n",
      "73:\tlearn: 0.0040158\ttotal: 779ms\tremaining: 274ms\n",
      "74:\tlearn: 0.0038867\ttotal: 788ms\tremaining: 263ms\n",
      "75:\tlearn: 0.0037618\ttotal: 798ms\tremaining: 252ms\n",
      "76:\tlearn: 0.0036454\ttotal: 807ms\tremaining: 241ms\n",
      "77:\tlearn: 0.0035582\ttotal: 816ms\tremaining: 230ms\n",
      "78:\tlearn: 0.0034622\ttotal: 825ms\tremaining: 219ms\n",
      "79:\tlearn: 0.0033738\ttotal: 834ms\tremaining: 208ms\n",
      "80:\tlearn: 0.0033126\ttotal: 843ms\tremaining: 198ms\n",
      "81:\tlearn: 0.0031680\ttotal: 852ms\tremaining: 187ms\n",
      "82:\tlearn: 0.0030961\ttotal: 862ms\tremaining: 176ms\n",
      "83:\tlearn: 0.0030246\ttotal: 871ms\tremaining: 166ms\n",
      "84:\tlearn: 0.0029450\ttotal: 881ms\tremaining: 155ms\n",
      "85:\tlearn: 0.0028060\ttotal: 891ms\tremaining: 145ms\n",
      "86:\tlearn: 0.0027338\ttotal: 901ms\tremaining: 135ms\n",
      "87:\tlearn: 0.0026297\ttotal: 910ms\tremaining: 124ms\n",
      "88:\tlearn: 0.0025807\ttotal: 919ms\tremaining: 114ms\n",
      "89:\tlearn: 0.0024927\ttotal: 929ms\tremaining: 103ms\n",
      "90:\tlearn: 0.0024843\ttotal: 950ms\tremaining: 94ms\n",
      "91:\tlearn: 0.0024246\ttotal: 961ms\tremaining: 83.5ms\n",
      "92:\tlearn: 0.0023688\ttotal: 971ms\tremaining: 73.1ms\n",
      "93:\tlearn: 0.0023293\ttotal: 982ms\tremaining: 62.7ms\n",
      "94:\tlearn: 0.0022904\ttotal: 993ms\tremaining: 52.3ms\n",
      "95:\tlearn: 0.0022169\ttotal: 1s\tremaining: 41.8ms\n",
      "96:\tlearn: 0.0021798\ttotal: 1.01s\tremaining: 31.3ms\n",
      "97:\tlearn: 0.0021506\ttotal: 1.02s\tremaining: 20.9ms\n",
      "98:\tlearn: 0.0020880\ttotal: 1.03s\tremaining: 10.4ms\n",
      "99:\tlearn: 0.0020534\ttotal: 1.04s\tremaining: 0us\n",
      "0:\tlearn: 1.4092431\ttotal: 20.6ms\tremaining: 2.04s\n",
      "1:\tlearn: 1.1331731\ttotal: 39.8ms\tremaining: 1.95s\n",
      "2:\tlearn: 0.9437287\ttotal: 57.5ms\tremaining: 1.86s\n",
      "3:\tlearn: 0.8026871\ttotal: 73ms\tremaining: 1.75s\n",
      "4:\tlearn: 0.6926710\ttotal: 88.3ms\tremaining: 1.68s\n",
      "5:\tlearn: 0.6012278\ttotal: 104ms\tremaining: 1.63s\n",
      "6:\tlearn: 0.5262525\ttotal: 118ms\tremaining: 1.56s\n",
      "7:\tlearn: 0.4619845\ttotal: 132ms\tremaining: 1.51s\n",
      "8:\tlearn: 0.4097921\ttotal: 145ms\tremaining: 1.47s\n",
      "9:\tlearn: 0.3642185\ttotal: 158ms\tremaining: 1.42s\n",
      "10:\tlearn: 0.3250034\ttotal: 171ms\tremaining: 1.38s\n",
      "11:\tlearn: 0.2908790\ttotal: 184ms\tremaining: 1.35s\n",
      "12:\tlearn: 0.2614940\ttotal: 197ms\tremaining: 1.31s\n",
      "13:\tlearn: 0.2356998\ttotal: 210ms\tremaining: 1.29s\n",
      "14:\tlearn: 0.2120750\ttotal: 222ms\tremaining: 1.26s\n",
      "15:\tlearn: 0.1915615\ttotal: 235ms\tremaining: 1.24s\n",
      "16:\tlearn: 0.1730097\ttotal: 247ms\tremaining: 1.21s\n",
      "17:\tlearn: 0.1564121\ttotal: 259ms\tremaining: 1.18s\n",
      "18:\tlearn: 0.1421931\ttotal: 269ms\tremaining: 1.15s\n",
      "19:\tlearn: 0.1291123\ttotal: 280ms\tremaining: 1.12s\n",
      "20:\tlearn: 0.1171349\ttotal: 291ms\tremaining: 1.09s\n",
      "21:\tlearn: 0.1065483\ttotal: 301ms\tremaining: 1.07s\n",
      "22:\tlearn: 0.0970240\ttotal: 311ms\tremaining: 1.04s\n",
      "23:\tlearn: 0.0884240\ttotal: 321ms\tremaining: 1.02s\n",
      "24:\tlearn: 0.0803334\ttotal: 331ms\tremaining: 994ms\n",
      "25:\tlearn: 0.0730508\ttotal: 341ms\tremaining: 971ms\n",
      "26:\tlearn: 0.0661975\ttotal: 351ms\tremaining: 949ms\n",
      "27:\tlearn: 0.0602781\ttotal: 361ms\tremaining: 927ms\n",
      "28:\tlearn: 0.0552132\ttotal: 370ms\tremaining: 907ms\n",
      "29:\tlearn: 0.0504049\ttotal: 380ms\tremaining: 887ms\n",
      "30:\tlearn: 0.0462356\ttotal: 390ms\tremaining: 868ms\n",
      "31:\tlearn: 0.0423082\ttotal: 400ms\tremaining: 849ms\n",
      "32:\tlearn: 0.0388792\ttotal: 410ms\tremaining: 832ms\n",
      "33:\tlearn: 0.0357143\ttotal: 419ms\tremaining: 813ms\n",
      "34:\tlearn: 0.0328566\ttotal: 429ms\tremaining: 796ms\n",
      "35:\tlearn: 0.0302547\ttotal: 438ms\tremaining: 779ms\n",
      "36:\tlearn: 0.0278380\ttotal: 448ms\tremaining: 763ms\n",
      "37:\tlearn: 0.0257833\ttotal: 457ms\tremaining: 746ms\n",
      "38:\tlearn: 0.0239441\ttotal: 467ms\tremaining: 730ms\n",
      "39:\tlearn: 0.0221297\ttotal: 477ms\tremaining: 715ms\n",
      "40:\tlearn: 0.0204927\ttotal: 487ms\tremaining: 701ms\n",
      "41:\tlearn: 0.0189243\ttotal: 498ms\tremaining: 687ms\n",
      "42:\tlearn: 0.0175169\ttotal: 507ms\tremaining: 672ms\n",
      "43:\tlearn: 0.0165020\ttotal: 516ms\tremaining: 657ms\n",
      "44:\tlearn: 0.0155405\ttotal: 526ms\tremaining: 643ms\n",
      "45:\tlearn: 0.0146957\ttotal: 536ms\tremaining: 630ms\n",
      "46:\tlearn: 0.0137973\ttotal: 547ms\tremaining: 617ms\n",
      "47:\tlearn: 0.0129474\ttotal: 557ms\tremaining: 603ms\n",
      "48:\tlearn: 0.0121701\ttotal: 566ms\tremaining: 589ms\n",
      "49:\tlearn: 0.0114666\ttotal: 576ms\tremaining: 576ms\n",
      "50:\tlearn: 0.0108142\ttotal: 586ms\tremaining: 563ms\n",
      "51:\tlearn: 0.0102595\ttotal: 595ms\tremaining: 549ms\n",
      "52:\tlearn: 0.0096143\ttotal: 605ms\tremaining: 537ms\n",
      "53:\tlearn: 0.0091430\ttotal: 615ms\tremaining: 524ms\n",
      "54:\tlearn: 0.0087962\ttotal: 624ms\tremaining: 510ms\n",
      "55:\tlearn: 0.0084731\ttotal: 633ms\tremaining: 497ms\n",
      "56:\tlearn: 0.0080008\ttotal: 642ms\tremaining: 484ms\n",
      "57:\tlearn: 0.0076593\ttotal: 652ms\tremaining: 472ms\n",
      "58:\tlearn: 0.0073625\ttotal: 661ms\tremaining: 459ms\n",
      "59:\tlearn: 0.0069498\ttotal: 670ms\tremaining: 447ms\n",
      "60:\tlearn: 0.0067044\ttotal: 680ms\tremaining: 434ms\n",
      "61:\tlearn: 0.0063225\ttotal: 689ms\tremaining: 423ms\n",
      "62:\tlearn: 0.0061378\ttotal: 699ms\tremaining: 410ms\n",
      "63:\tlearn: 0.0058816\ttotal: 708ms\tremaining: 398ms\n",
      "64:\tlearn: 0.0055998\ttotal: 718ms\tremaining: 387ms\n",
      "65:\tlearn: 0.0054094\ttotal: 727ms\tremaining: 375ms\n",
      "66:\tlearn: 0.0051976\ttotal: 737ms\tremaining: 363ms\n",
      "67:\tlearn: 0.0050727\ttotal: 747ms\tremaining: 352ms\n",
      "68:\tlearn: 0.0047871\ttotal: 757ms\tremaining: 340ms\n",
      "69:\tlearn: 0.0046101\ttotal: 767ms\tremaining: 329ms\n",
      "70:\tlearn: 0.0043475\ttotal: 778ms\tremaining: 318ms\n",
      "71:\tlearn: 0.0042556\ttotal: 784ms\tremaining: 305ms\n",
      "72:\tlearn: 0.0041161\ttotal: 795ms\tremaining: 294ms\n",
      "73:\tlearn: 0.0040707\ttotal: 805ms\tremaining: 283ms\n",
      "74:\tlearn: 0.0039640\ttotal: 814ms\tremaining: 271ms\n",
      "75:\tlearn: 0.0038319\ttotal: 824ms\tremaining: 260ms\n",
      "76:\tlearn: 0.0037723\ttotal: 833ms\tremaining: 249ms\n",
      "77:\tlearn: 0.0036593\ttotal: 842ms\tremaining: 237ms\n",
      "78:\tlearn: 0.0035826\ttotal: 851ms\tremaining: 226ms\n",
      "79:\tlearn: 0.0034263\ttotal: 861ms\tremaining: 215ms\n",
      "80:\tlearn: 0.0033476\ttotal: 870ms\tremaining: 204ms\n",
      "81:\tlearn: 0.0033181\ttotal: 879ms\tremaining: 193ms\n",
      "82:\tlearn: 0.0032643\ttotal: 887ms\tremaining: 182ms\n",
      "83:\tlearn: 0.0032170\ttotal: 896ms\tremaining: 171ms\n",
      "84:\tlearn: 0.0030516\ttotal: 906ms\tremaining: 160ms\n",
      "85:\tlearn: 0.0030028\ttotal: 915ms\tremaining: 149ms\n",
      "86:\tlearn: 0.0029657\ttotal: 924ms\tremaining: 138ms\n",
      "87:\tlearn: 0.0028645\ttotal: 934ms\tremaining: 127ms\n",
      "88:\tlearn: 0.0027929\ttotal: 943ms\tremaining: 117ms\n",
      "89:\tlearn: 0.0027242\ttotal: 953ms\tremaining: 106ms\n",
      "90:\tlearn: 0.0026845\ttotal: 961ms\tremaining: 95.1ms\n",
      "91:\tlearn: 0.0026375\ttotal: 971ms\tremaining: 84.4ms\n",
      "92:\tlearn: 0.0025586\ttotal: 981ms\tremaining: 73.8ms\n",
      "93:\tlearn: 0.0024565\ttotal: 992ms\tremaining: 63.3ms\n",
      "94:\tlearn: 0.0024204\ttotal: 1s\tremaining: 52.7ms\n",
      "95:\tlearn: 0.0023818\ttotal: 1.01s\tremaining: 42.1ms\n",
      "96:\tlearn: 0.0023000\ttotal: 1.02s\tremaining: 31.6ms\n",
      "97:\tlearn: 0.0022349\ttotal: 1.03s\tremaining: 21.1ms\n",
      "98:\tlearn: 0.0021445\ttotal: 1.04s\tremaining: 10.5ms\n",
      "99:\tlearn: 0.0020459\ttotal: 1.05s\tremaining: 0us\n",
      "0:\tlearn: 1.4086753\ttotal: 21.2ms\tremaining: 2.1s\n",
      "1:\tlearn: 1.1328079\ttotal: 39.2ms\tremaining: 1.92s\n",
      "2:\tlearn: 0.9417284\ttotal: 53.2ms\tremaining: 1.72s\n",
      "3:\tlearn: 0.8024912\ttotal: 67ms\tremaining: 1.61s\n",
      "4:\tlearn: 0.6929099\ttotal: 80ms\tremaining: 1.52s\n",
      "5:\tlearn: 0.6042117\ttotal: 93.5ms\tremaining: 1.46s\n",
      "6:\tlearn: 0.5285628\ttotal: 105ms\tremaining: 1.4s\n",
      "7:\tlearn: 0.4636784\ttotal: 116ms\tremaining: 1.34s\n",
      "8:\tlearn: 0.4112991\ttotal: 128ms\tremaining: 1.29s\n",
      "9:\tlearn: 0.3655823\ttotal: 139ms\tremaining: 1.25s\n",
      "10:\tlearn: 0.3262653\ttotal: 150ms\tremaining: 1.21s\n",
      "11:\tlearn: 0.2918888\ttotal: 161ms\tremaining: 1.18s\n",
      "12:\tlearn: 0.2615697\ttotal: 171ms\tremaining: 1.15s\n",
      "13:\tlearn: 0.2357083\ttotal: 204ms\tremaining: 1.25s\n",
      "14:\tlearn: 0.2115623\ttotal: 214ms\tremaining: 1.21s\n",
      "15:\tlearn: 0.1909720\ttotal: 223ms\tremaining: 1.17s\n",
      "16:\tlearn: 0.1724683\ttotal: 233ms\tremaining: 1.14s\n",
      "17:\tlearn: 0.1557437\ttotal: 243ms\tremaining: 1.1s\n",
      "18:\tlearn: 0.1413516\ttotal: 253ms\tremaining: 1.08s\n",
      "19:\tlearn: 0.1282119\ttotal: 262ms\tremaining: 1.05s\n",
      "20:\tlearn: 0.1163471\ttotal: 273ms\tremaining: 1.03s\n",
      "21:\tlearn: 0.1057849\ttotal: 284ms\tremaining: 1.01s\n",
      "22:\tlearn: 0.0962792\ttotal: 294ms\tremaining: 985ms\n",
      "23:\tlearn: 0.0876485\ttotal: 304ms\tremaining: 962ms\n",
      "24:\tlearn: 0.0796302\ttotal: 314ms\tremaining: 942ms\n",
      "25:\tlearn: 0.0722302\ttotal: 324ms\tremaining: 923ms\n",
      "26:\tlearn: 0.0656291\ttotal: 335ms\tremaining: 906ms\n",
      "27:\tlearn: 0.0600778\ttotal: 345ms\tremaining: 887ms\n",
      "28:\tlearn: 0.0547755\ttotal: 355ms\tremaining: 868ms\n",
      "29:\tlearn: 0.0500502\ttotal: 365ms\tremaining: 851ms\n",
      "30:\tlearn: 0.0458272\ttotal: 374ms\tremaining: 832ms\n",
      "31:\tlearn: 0.0420425\ttotal: 383ms\tremaining: 815ms\n",
      "32:\tlearn: 0.0388258\ttotal: 393ms\tremaining: 797ms\n",
      "33:\tlearn: 0.0357996\ttotal: 402ms\tremaining: 781ms\n",
      "34:\tlearn: 0.0329842\ttotal: 413ms\tremaining: 767ms\n",
      "35:\tlearn: 0.0304938\ttotal: 422ms\tremaining: 750ms\n",
      "36:\tlearn: 0.0281347\ttotal: 432ms\tremaining: 735ms\n",
      "37:\tlearn: 0.0257929\ttotal: 442ms\tremaining: 721ms\n",
      "38:\tlearn: 0.0237168\ttotal: 451ms\tremaining: 706ms\n",
      "39:\tlearn: 0.0220999\ttotal: 460ms\tremaining: 690ms\n",
      "40:\tlearn: 0.0203742\ttotal: 469ms\tremaining: 676ms\n",
      "41:\tlearn: 0.0187695\ttotal: 479ms\tremaining: 661ms\n",
      "42:\tlearn: 0.0174940\ttotal: 488ms\tremaining: 647ms\n",
      "43:\tlearn: 0.0162327\ttotal: 498ms\tremaining: 634ms\n",
      "44:\tlearn: 0.0150682\ttotal: 508ms\tremaining: 621ms\n",
      "45:\tlearn: 0.0141083\ttotal: 518ms\tremaining: 608ms\n",
      "46:\tlearn: 0.0133147\ttotal: 528ms\tremaining: 595ms\n",
      "47:\tlearn: 0.0124197\ttotal: 539ms\tremaining: 583ms\n",
      "48:\tlearn: 0.0115886\ttotal: 549ms\tremaining: 571ms\n",
      "49:\tlearn: 0.0108583\ttotal: 559ms\tremaining: 559ms\n",
      "50:\tlearn: 0.0103175\ttotal: 569ms\tremaining: 546ms\n",
      "51:\tlearn: 0.0096928\ttotal: 579ms\tremaining: 535ms\n",
      "52:\tlearn: 0.0090758\ttotal: 590ms\tremaining: 523ms\n",
      "53:\tlearn: 0.0086456\ttotal: 600ms\tremaining: 511ms\n",
      "54:\tlearn: 0.0082128\ttotal: 610ms\tremaining: 499ms\n",
      "55:\tlearn: 0.0077671\ttotal: 619ms\tremaining: 486ms\n",
      "56:\tlearn: 0.0073463\ttotal: 629ms\tremaining: 474ms\n",
      "57:\tlearn: 0.0069272\ttotal: 637ms\tremaining: 462ms\n",
      "58:\tlearn: 0.0066364\ttotal: 647ms\tremaining: 449ms\n",
      "59:\tlearn: 0.0063155\ttotal: 656ms\tremaining: 437ms\n",
      "60:\tlearn: 0.0060644\ttotal: 666ms\tremaining: 425ms\n",
      "61:\tlearn: 0.0058088\ttotal: 675ms\tremaining: 414ms\n",
      "62:\tlearn: 0.0055880\ttotal: 684ms\tremaining: 402ms\n",
      "63:\tlearn: 0.0053756\ttotal: 693ms\tremaining: 390ms\n",
      "64:\tlearn: 0.0051329\ttotal: 702ms\tremaining: 378ms\n",
      "65:\tlearn: 0.0049445\ttotal: 711ms\tremaining: 366ms\n",
      "66:\tlearn: 0.0047276\ttotal: 721ms\tremaining: 355ms\n",
      "67:\tlearn: 0.0045462\ttotal: 731ms\tremaining: 344ms\n",
      "68:\tlearn: 0.0043345\ttotal: 741ms\tremaining: 333ms\n",
      "69:\tlearn: 0.0042280\ttotal: 750ms\tremaining: 322ms\n",
      "70:\tlearn: 0.0040545\ttotal: 760ms\tremaining: 310ms\n",
      "71:\tlearn: 0.0039501\ttotal: 769ms\tremaining: 299ms\n",
      "72:\tlearn: 0.0038033\ttotal: 780ms\tremaining: 289ms\n",
      "73:\tlearn: 0.0036918\ttotal: 791ms\tremaining: 278ms\n",
      "74:\tlearn: 0.0035930\ttotal: 802ms\tremaining: 267ms\n",
      "75:\tlearn: 0.0034507\ttotal: 813ms\tremaining: 257ms\n",
      "76:\tlearn: 0.0033073\ttotal: 824ms\tremaining: 246ms\n",
      "77:\tlearn: 0.0032018\ttotal: 833ms\tremaining: 235ms\n",
      "78:\tlearn: 0.0031297\ttotal: 844ms\tremaining: 224ms\n",
      "79:\tlearn: 0.0030624\ttotal: 854ms\tremaining: 214ms\n",
      "80:\tlearn: 0.0029847\ttotal: 864ms\tremaining: 203ms\n",
      "81:\tlearn: 0.0028466\ttotal: 874ms\tremaining: 192ms\n",
      "82:\tlearn: 0.0028046\ttotal: 884ms\tremaining: 181ms\n",
      "83:\tlearn: 0.0027566\ttotal: 893ms\tremaining: 170ms\n",
      "84:\tlearn: 0.0026712\ttotal: 904ms\tremaining: 159ms\n",
      "85:\tlearn: 0.0025889\ttotal: 911ms\tremaining: 148ms\n",
      "86:\tlearn: 0.0025411\ttotal: 921ms\tremaining: 138ms\n",
      "87:\tlearn: 0.0024742\ttotal: 931ms\tremaining: 127ms\n",
      "88:\tlearn: 0.0024343\ttotal: 940ms\tremaining: 116ms\n",
      "89:\tlearn: 0.0023913\ttotal: 950ms\tremaining: 106ms\n",
      "90:\tlearn: 0.0023586\ttotal: 960ms\tremaining: 94.9ms\n",
      "91:\tlearn: 0.0023072\ttotal: 969ms\tremaining: 84.3ms\n",
      "92:\tlearn: 0.0022421\ttotal: 980ms\tremaining: 73.8ms\n",
      "93:\tlearn: 0.0022118\ttotal: 989ms\tremaining: 63.2ms\n",
      "94:\tlearn: 0.0021686\ttotal: 999ms\tremaining: 52.6ms\n",
      "95:\tlearn: 0.0020880\ttotal: 1.01s\tremaining: 42ms\n",
      "96:\tlearn: 0.0020244\ttotal: 1.02s\tremaining: 31.5ms\n",
      "97:\tlearn: 0.0019646\ttotal: 1.03s\tremaining: 21ms\n",
      "98:\tlearn: 0.0019202\ttotal: 1.04s\tremaining: 10.5ms\n",
      "99:\tlearn: 0.0018804\ttotal: 1.05s\tremaining: 0us\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0    2.0    3.0    4.0    5.0      6.0\n",
      "0.0  2337.0     0.0    0.0    0.0    0.0    0.0      0.0\n",
      "1.0     0.0  5480.0    0.0    0.0    0.0    0.0      1.0\n",
      "2.0     0.0     0.0  190.0    0.0    0.0    0.0      0.0\n",
      "3.0     0.0     0.0    0.0  189.0    0.0    0.0      0.0\n",
      "4.0     2.0     0.0    0.0    0.0  165.0    0.0      0.0\n",
      "5.0     0.0     0.0    3.0    0.0    0.0  214.0      0.0\n",
      "6.0     0.0     0.0    0.0    0.0    0.0    0.0  16313.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9997589780670041\n",
      "Precision total:  0.9976485136051577\n",
      "Recall total:  0.9962880884064101\n",
      "F1 total:  0.996947550083876\n",
      "BACC total:  0.9962880884064101\n",
      "MCC total:  0.9995302472291382\n"
     ]
    }
   ],
   "source": [
    "import catboost\n",
    "start = time.time()\n",
    "\n",
    "bag_cat = catboost.CatBoostClassifier(iterations=100, depth=6, learning_rate=0.1, loss_function='MultiClass', custom_metric='Accuracy')\n",
    "\n",
    "base_classifier = bag_cat\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_cat'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Baggin LGBM\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0    2.0    3.0    4.0    5.0      6.0\n",
      "0.0  2337.0     0.0    0.0    0.0    0.0    0.0      0.0\n",
      "1.0     0.0  5481.0    0.0    0.0    0.0    0.0      0.0\n",
      "2.0     0.0     0.0  190.0    0.0    0.0    0.0      0.0\n",
      "3.0     0.0     0.0    0.0  189.0    0.0    0.0      0.0\n",
      "4.0     0.0     0.0    0.0    0.0  167.0    0.0      0.0\n",
      "5.0     0.0     0.0    0.0    0.0    0.0  217.0      0.0\n",
      "6.0     0.0     0.0    0.0    0.0    0.0    0.0  16313.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  1.0\n",
      "Precision total:  1.0\n",
      "Recall total:  1.0\n",
      "F1 total:  1.0\n",
      "BACC total:  1.0\n",
      "MCC total:  1.0\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "from lightgbm import LGBMClassifier\n",
    "lgbm = LGBMClassifier()\n",
    "\n",
    "\n",
    "base_classifier = lgbm\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_lgbm'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# import xgboost as xgb\n",
    "\n",
    "# # Create a DMatrix for XGBoost\n",
    "# dtrain = xgb.DMatrix(X_train_01, label=y_train_01)\n",
    "# dtest = xgb.DMatrix(X_test_01, label=y_test_01)\n",
    "\n",
    "# # Set XGBoost parameters\n",
    "# params = {\n",
    "#     'objective': 'multi:softmax',  # for multi-class classification\n",
    "#     'num_class': 5,  # specify the number of classes\n",
    "#     'max_depth': 3,\n",
    "#     'learning_rate': 0.1,\n",
    "#     'eval_metric': 'mlogloss'  # metric for multi-class classification\n",
    "# }\n",
    "\n",
    "# # Train the XGBoost model\n",
    "# num_round = 100\n",
    "# xgb_01 = xgb.train(params, dtrain, num_round)\n",
    "\n",
    "# base_classifier = xgb\n",
    "\n",
    "# # Define the BaggingClassifier\n",
    "# bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# # Train the BaggingClassifier\n",
    "# bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# # Make predictions on the test set\n",
    "# y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "# name = 'bag_xgb'\n",
    "\n",
    "# pred_label = y_pred\n",
    "\n",
    "\n",
    "# metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "# Acc = metrics[0]\n",
    "# Precision = metrics[1]\n",
    "# Recall = metrics[2]\n",
    "# F1 = metrics[3]\n",
    "# BACC = metrics[4]\n",
    "# MCC = metrics[5]    \n",
    "\n",
    "\n",
    "# globals()[f\"{name}_acc_01\"] = Acc\n",
    "# globals()[f\"{name}_pre_01\"] = Precision\n",
    "# globals()[f\"{name}_rec_01\"] = Recall\n",
    "# globals()[f\"{name}_f1_01\"] = F1\n",
    "# globals()[f\"{name}_bacc_01\"] = BACC\n",
    "# globals()[f\"{name}_mcc_01\"] = MCC\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0    2.0    3.0    4.0    5.0      6.0\n",
      "0.0  2321.0     9.0    4.0    1.0    2.0    0.0      0.0\n",
      "1.0     0.0  5469.0    0.0    0.0    0.0    0.0     12.0\n",
      "2.0     0.0     0.0  190.0    0.0    0.0    0.0      0.0\n",
      "3.0     0.0     0.0    0.0  189.0    0.0    0.0      0.0\n",
      "4.0     0.0     0.0   17.0    0.0  150.0    0.0      0.0\n",
      "5.0     0.0     0.0    9.0    0.0    0.0  208.0      0.0\n",
      "6.0     0.0     0.0    0.0    0.0    0.0    0.0  16313.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9978308026030369\n",
      "Precision total:  0.9775481866734569\n",
      "Recall total:  0.9782418818119212\n",
      "F1 total:  0.9768191847581741\n",
      "BACC total:  0.9782418818119212\n",
      "MCC total:  0.9957727465007264\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rf = RandomForestClassifier(max_depth = 5,  n_estimators = 10, min_samples_split = 2, n_jobs = -1)\n",
    "\n",
    "base_classifier = rf\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_rf'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging with many models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### do bootstrapping "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 1. Multiple subsets are created from the original dataset, selecting observations with replacement.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "\n",
    "num_bootstraps = 10  # Adjust the number of bootstraps as needed\n",
    "\n",
    "original_data_df = X_train_01.assign(label = y_train_01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "boot_df = []\n",
    "for i in range(0,num_bootstraps): \n",
    "    boot_df.append(original_data_df.sample(frac = 1, replace=True).reset_index(drop=True))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dnn</th>\n",
       "      <th>rf</th>\n",
       "      <th>lgbm</th>\n",
       "      <th>ada</th>\n",
       "      <th>knn</th>\n",
       "      <th>mlp</th>\n",
       "      <th>svm</th>\n",
       "      <th>cat</th>\n",
       "      <th>xgb</th>\n",
       "      <th>lr</th>\n",
       "      <th>dt</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.820344</td>\n",
       "      <td>0.981712</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.177443</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.358213</td>\n",
       "      <td>0.998089</td>\n",
       "      <td>0.996417</td>\n",
       "      <td>0.993499</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.218659</td>\n",
       "      <td>0.974561</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.170730</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999877</td>\n",
       "      <td>0.036689</td>\n",
       "      <td>0.995759</td>\n",
       "      <td>0.998961</td>\n",
       "      <td>0.946956</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.360583</td>\n",
       "      <td>0.817928</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.170281</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.996568</td>\n",
       "      <td>0.417810</td>\n",
       "      <td>0.992388</td>\n",
       "      <td>0.992857</td>\n",
       "      <td>0.540728</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.356358</td>\n",
       "      <td>0.817928</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.170281</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.997997</td>\n",
       "      <td>0.637038</td>\n",
       "      <td>0.992881</td>\n",
       "      <td>0.992857</td>\n",
       "      <td>0.519779</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.825415</td>\n",
       "      <td>0.927554</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.226642</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.271906</td>\n",
       "      <td>0.998049</td>\n",
       "      <td>0.984435</td>\n",
       "      <td>0.995693</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58080</th>\n",
       "      <td>0.816978</td>\n",
       "      <td>0.981712</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.177443</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.318344</td>\n",
       "      <td>0.998673</td>\n",
       "      <td>0.996417</td>\n",
       "      <td>0.998313</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58081</th>\n",
       "      <td>0.998733</td>\n",
       "      <td>0.989548</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.221467</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.470425</td>\n",
       "      <td>0.997100</td>\n",
       "      <td>0.991007</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58082</th>\n",
       "      <td>0.812853</td>\n",
       "      <td>0.981712</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.177443</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.263367</td>\n",
       "      <td>0.998846</td>\n",
       "      <td>0.996581</td>\n",
       "      <td>0.999789</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58083</th>\n",
       "      <td>0.810734</td>\n",
       "      <td>0.981712</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.177443</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.339167</td>\n",
       "      <td>0.997452</td>\n",
       "      <td>0.996261</td>\n",
       "      <td>0.997669</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58084</th>\n",
       "      <td>0.813960</td>\n",
       "      <td>0.981712</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.177443</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.339437</td>\n",
       "      <td>0.999040</td>\n",
       "      <td>0.997243</td>\n",
       "      <td>0.997813</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>58085 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            dnn        rf  lgbm       ada  knn       mlp       svm       cat  \\\n",
       "0      0.820344  0.981712   1.0  0.177443  1.0  0.999999  0.358213  0.998089   \n",
       "1      0.218659  0.974561   0.0  0.170730  1.0  0.999877  0.036689  0.995759   \n",
       "2      0.360583  0.817928   1.0  0.170281  1.0  0.996568  0.417810  0.992388   \n",
       "3      0.356358  0.817928   1.0  0.170281  1.0  0.997997  0.637038  0.992881   \n",
       "4      0.825415  0.927554   1.0  0.226642  1.0  1.000000  0.271906  0.998049   \n",
       "...         ...       ...   ...       ...  ...       ...       ...       ...   \n",
       "58080  0.816978  0.981712   1.0  0.177443  1.0  0.999999  0.318344  0.998673   \n",
       "58081  0.998733  0.989548   1.0  0.221467  1.0  1.000000  0.470425  0.997100   \n",
       "58082  0.812853  0.981712   1.0  0.177443  1.0  1.000000  0.263367  0.998846   \n",
       "58083  0.810734  0.981712   1.0  0.177443  1.0  1.000000  0.339167  0.997452   \n",
       "58084  0.813960  0.981712   1.0  0.177443  1.0  0.999999  0.339437  0.999040   \n",
       "\n",
       "            xgb        lr   dt  label  \n",
       "0      0.996417  0.993499  1.0    6.0  \n",
       "1      0.998961  0.946956  1.0    0.0  \n",
       "2      0.992857  0.540728  1.0    1.0  \n",
       "3      0.992857  0.519779  1.0    1.0  \n",
       "4      0.984435  0.995693  1.0    6.0  \n",
       "...         ...       ...  ...    ...  \n",
       "58080  0.996417  0.998313  1.0    6.0  \n",
       "58081  0.991007  1.000000  1.0    1.0  \n",
       "58082  0.996581  0.999789  1.0    6.0  \n",
       "58083  0.996261  0.997669  1.0    6.0  \n",
       "58084  0.997243  0.997813  1.0    6.0  \n",
       "\n",
       "[58085 rows x 12 columns]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "boot_df[5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.A base model (weak model) is created on each of these subsets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bag_comb_pred = []\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SVM\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "clf = SGDClassifier(\n",
    "    loss='hinge',           # hinge loss for linear SVM\n",
    "    penalty='l2',           # L2 regularization to prevent overfitting\n",
    "    alpha=1e-4,             # Learning rate (small value for fine-grained updates)\n",
    "    max_iter=1000,          # Number of passes over the training data\n",
    "    random_state=42,        # Seed for reproducible results\n",
    "    learning_rate='optimal' # Automatically adjusts the learning rate based on the training data\n",
    ")\n",
    "y_train_boot = boot_df[0].pop('label')\n",
    "X_train_boot = boot_df[0]\n",
    "clf.fit(X_train_boot, y_train_boot)\n",
    "preds_svm_01 = clf.predict(X_test_01)\n",
    "bag_comb_pred.append(preds_svm_01)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#ADA\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "abc = AdaBoostClassifier(n_estimators=50, learning_rate=1.0)\n",
    "ada = abc.fit(X_train_01, y_train_01)\n",
    "y_train_boot = boot_df[1].pop('label')\n",
    "X_train_boot = boot_df[1]\n",
    "preds_ada_01 = ada.predict(X_test_01)\n",
    "bag_comb_pred.append(preds_ada_01)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 1.4057894\ttest: 1.4052880\tbest: 1.4052880 (0)\ttotal: 24.8ms\tremaining: 2.46s\n",
      "10:\tlearn: 0.3241261\ttest: 0.3236720\tbest: 0.3236720 (10)\ttotal: 158ms\tremaining: 1.27s\n",
      "20:\tlearn: 0.1147097\ttest: 0.1144330\tbest: 0.1144330 (20)\ttotal: 274ms\tremaining: 1.03s\n",
      "30:\tlearn: 0.0443521\ttest: 0.0444153\tbest: 0.0444153 (30)\ttotal: 374ms\tremaining: 832ms\n",
      "40:\tlearn: 0.0196613\ttest: 0.0198684\tbest: 0.0198684 (40)\ttotal: 469ms\tremaining: 675ms\n",
      "50:\tlearn: 0.0099918\ttest: 0.0102880\tbest: 0.0102880 (50)\ttotal: 567ms\tremaining: 545ms\n",
      "60:\tlearn: 0.0061498\ttest: 0.0064829\tbest: 0.0064829 (60)\ttotal: 661ms\tremaining: 423ms\n",
      "70:\tlearn: 0.0040792\ttest: 0.0044397\tbest: 0.0044397 (70)\ttotal: 758ms\tremaining: 310ms\n",
      "80:\tlearn: 0.0028003\ttest: 0.0031882\tbest: 0.0031882 (80)\ttotal: 855ms\tremaining: 201ms\n",
      "90:\tlearn: 0.0021878\ttest: 0.0025858\tbest: 0.0025858 (90)\ttotal: 951ms\tremaining: 94ms\n",
      "99:\tlearn: 0.0017170\ttest: 0.0021212\tbest: 0.0021212 (99)\ttotal: 1.04s\tremaining: 0us\n",
      "\n",
      "bestTest = 0.0021211588\n",
      "bestIteration = 99\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Catboost\n",
    "import catboost\n",
    "cat_01 = catboost.CatBoostClassifier(iterations=100, depth=6, learning_rate=0.1, loss_function='MultiClass', custom_metric='Accuracy')\n",
    "y_train_boot = boot_df[2].pop('label')\n",
    "X_train_boot = boot_df[2]\n",
    "cat_01.fit(X_train_boot, y_train_boot, eval_set=(X_test_01, y_test_01), verbose=10)\n",
    "preds_cat = cat_01.predict(X_test_01)\n",
    "preds_cat = np.squeeze(preds_cat)\n",
    "pred_label = preds_cat\n",
    "bag_comb_pred.append(preds_cat)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n"
     ]
    }
   ],
   "source": [
    "#MLP\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "mlp = MLPClassifier(hidden_layer_sizes=(100,), max_iter=200, random_state=1)\n",
    "y_train_boot = boot_df[3].pop('label')\n",
    "X_train_boot = boot_df[3]\n",
    "if 1 == 1 and 0 == 0:\n",
    "    MLP = mlp.fit(X_train_boot, y_train_boot)\n",
    "    y_pred = MLP.predict_proba(X_test_01)\n",
    "    preds_mlp_01 = np.argmax(y_pred,axis = 1)\n",
    "\n",
    "bag_comb_pred.append(preds_mlp_01)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining LGBM Model\n",
      "---------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#LGBM\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining LGBM Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "#LGBM\n",
    "from lightgbm import LGBMClassifier\n",
    "lgbm = LGBMClassifier()\n",
    "y_train_boot = boot_df[4].pop('label')\n",
    "X_train_boot = boot_df[4]\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "    lgbm.fit(X_train_boot, y_train_boot)\n",
    "    preds_lgbm_01 = lgbm.predict(X_test_01)\n",
    "    bag_comb_pred.append(preds_lgbm_01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#KNN\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "knn_clf_01=KNeighborsClassifier(n_neighbors = 5)\n",
    "y_train_boot = boot_df[5].pop('label')\n",
    "X_train_boot = boot_df[5]\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "    knn_clf_01.fit(X_train_boot,y_train_boot)\n",
    "if use_model_knn == 1:\n",
    "    preds_knn =knn_clf_01.predict(X_test_01)\n",
    "    bag_comb_pred.append(preds_knn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Random Forest\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rf = RandomForestClassifier(max_depth = 5,  n_estimators = 10, min_samples_split = 2, n_jobs = -1)\n",
    "y_train_boot = boot_df[6].pop('label')\n",
    "X_train_boot = boot_df[6]\n",
    "\n",
    "if True == True:\n",
    "    model_rf_01 = rf.fit(X_train_boot,y_train_boot)\n",
    "    preds_rf_01 = model_rf_01.predict(X_test_01)\n",
    "    bag_comb_pred.append(preds_rf_01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "364/364 [==============================] - 1s 3ms/step - loss: 1.0309 - accuracy: 0.6273 - val_loss: 0.8294 - val_accuracy: 0.6660\n",
      "Epoch 2/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.7964 - accuracy: 0.6612 - val_loss: 0.7438 - val_accuracy: 0.6670\n",
      "Epoch 3/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.7409 - accuracy: 0.6635 - val_loss: 0.6977 - val_accuracy: 0.6754\n",
      "Epoch 4/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.6996 - accuracy: 0.6914 - val_loss: 0.6620 - val_accuracy: 0.7270\n",
      "Epoch 5/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.6651 - accuracy: 0.7261 - val_loss: 0.6324 - val_accuracy: 0.7502\n",
      "Epoch 6/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.6389 - accuracy: 0.7453 - val_loss: 0.6083 - val_accuracy: 0.7575\n",
      "Epoch 7/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.6141 - accuracy: 0.7529 - val_loss: 0.5826 - val_accuracy: 0.7574\n",
      "Epoch 8/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.5968 - accuracy: 0.7539 - val_loss: 0.5682 - val_accuracy: 0.7564\n",
      "Epoch 9/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.5849 - accuracy: 0.7556 - val_loss: 0.5584 - val_accuracy: 0.7579\n",
      "Epoch 10/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.5771 - accuracy: 0.7577 - val_loss: 0.5508 - val_accuracy: 0.7615\n",
      "Epoch 11/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.5683 - accuracy: 0.7603 - val_loss: 0.5439 - val_accuracy: 0.7637\n",
      "Epoch 12/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.5651 - accuracy: 0.7613 - val_loss: 0.5403 - val_accuracy: 0.7694\n",
      "Epoch 13/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.5592 - accuracy: 0.7656 - val_loss: 0.5345 - val_accuracy: 0.7709\n",
      "Epoch 14/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.5518 - accuracy: 0.7673 - val_loss: 0.5310 - val_accuracy: 0.7722\n",
      "Epoch 15/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.5518 - accuracy: 0.7684 - val_loss: 0.5323 - val_accuracy: 0.7723\n",
      "Epoch 16/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.5455 - accuracy: 0.7696 - val_loss: 0.5249 - val_accuracy: 0.7747\n",
      "Epoch 17/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.5455 - accuracy: 0.7697 - val_loss: 0.5223 - val_accuracy: 0.7739\n",
      "Epoch 18/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.5428 - accuracy: 0.7718 - val_loss: 0.5201 - val_accuracy: 0.7744\n",
      "Epoch 19/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.5405 - accuracy: 0.7729 - val_loss: 0.5193 - val_accuracy: 0.7771\n",
      "Epoch 20/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.5384 - accuracy: 0.7727 - val_loss: 0.5129 - val_accuracy: 0.7781\n",
      "Epoch 21/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.5320 - accuracy: 0.7721 - val_loss: 0.5086 - val_accuracy: 0.7787\n",
      "Epoch 22/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.5337 - accuracy: 0.7711 - val_loss: 0.5071 - val_accuracy: 0.7799\n",
      "Epoch 23/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.5275 - accuracy: 0.7732 - val_loss: 0.5000 - val_accuracy: 0.7824\n",
      "Epoch 24/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.5249 - accuracy: 0.7740 - val_loss: 0.4969 - val_accuracy: 0.7794\n",
      "Epoch 25/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.5197 - accuracy: 0.7736 - val_loss: 0.4895 - val_accuracy: 0.7733\n",
      "Epoch 26/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.5161 - accuracy: 0.7709 - val_loss: 0.4916 - val_accuracy: 0.7798\n",
      "Epoch 27/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.5126 - accuracy: 0.7706 - val_loss: 0.4848 - val_accuracy: 0.7779\n",
      "Epoch 28/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.5089 - accuracy: 0.7689 - val_loss: 0.4804 - val_accuracy: 0.7735\n",
      "Epoch 29/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.5069 - accuracy: 0.7687 - val_loss: 0.4757 - val_accuracy: 0.7689\n",
      "Epoch 30/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.5019 - accuracy: 0.7703 - val_loss: 0.4706 - val_accuracy: 0.7647\n",
      "Epoch 31/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.4967 - accuracy: 0.7733 - val_loss: 0.4706 - val_accuracy: 0.7622\n",
      "Epoch 32/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.4944 - accuracy: 0.7736 - val_loss: 0.4624 - val_accuracy: 0.7746\n",
      "Epoch 33/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.4933 - accuracy: 0.7751 - val_loss: 0.4619 - val_accuracy: 0.7763\n"
     ]
    }
   ],
   "source": [
    "#DNN\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "#Model Parameters\n",
    "y_train_boot = boot_df[7].pop('label')\n",
    "X_train_boot = boot_df[7]\n",
    "\n",
    "\n",
    "dropout_rate = 0.02\n",
    "nodes = 3\n",
    "out_layer = 7\n",
    "optimizer='adam'\n",
    "loss='sparse_categorical_crossentropy'\n",
    "epochs=100\n",
    "batch_size=128\n",
    "num_columns = X_train_boot.shape[1]\n",
    "dnn_01 = tf.keras.Sequential()\n",
    "# Input layer\n",
    "dnn_01.add(tf.keras.Input(shape=(num_columns,)))\n",
    "# Dense layers with dropout\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "# Output layer\n",
    "# dnn_01.add(tf.keras.layers.Dense(out_layer))\n",
    "dnn_01.add(tf.keras.layers.Dense(out_layer, activation='softmax'))\n",
    "# dnn.add(tf.keras.layers.Dense(out_layer, activation='softmax'))\n",
    "dnn_01.compile(optimizer=optimizer, loss=loss,metrics=['accuracy'])\n",
    "from keras.callbacks import EarlyStopping\n",
    "# Define EarlyStopping callback\n",
    "early_stopping = EarlyStopping(monitor='val_accuracy', patience=10, restore_best_weights=True)\n",
    "dnn_01.fit(X_train_boot, y_train_boot, epochs=epochs, batch_size=batch_size,validation_split=0.2, callbacks=[early_stopping])\n",
    "pred_dnn = dnn_01.predict(X_test_01)\n",
    "preds_dnn_01 = np.argmax(pred_dnn,axis = 1)\n",
    "bag_comb_pred.append(preds_dnn_01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n"
     ]
    }
   ],
   "source": [
    "#LogReg\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "logreg_01 = LogisticRegression()\n",
    "y_train_boot = boot_df[8].pop('label')\n",
    "X_train_boot = boot_df[8]\n",
    "\n",
    "logreg_01.fit(X_train_boot,y_train_boot)\n",
    "preds_logreg =logreg_01.predict(X_test_01)\n",
    "bag_comb_pred.append(preds_logreg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "y_train_boot = boot_df[9].pop('label')\n",
    "X_train_boot = boot_df[9]\n",
    "\n",
    "# Create a DMatrix for XGBoost\n",
    "dtrain = xgb.DMatrix(X_train_boot, label=y_train_boot)\n",
    "dtest = xgb.DMatrix(X_test_01, label=y_test_01)\n",
    "# Set XGBoost parameters\n",
    "params = {\n",
    "    'objective': 'multi:softmax',  # for multi-class classification\n",
    "    'num_class': 7,  # specify the number of classes\n",
    "    'max_depth': 3,\n",
    "    'learning_rate': 0.1,\n",
    "    'eval_metric': 'mlogloss'  # metric for multi-class classification\n",
    "}\n",
    "# Train the XGBoost model\n",
    "num_round = 100\n",
    "xgb_01 = xgb.train(params, dtrain, num_round)\n",
    "preds_xgb_01 = xgb_01.predict(dtest)\n",
    "bag_comb_pred.append(preds_xgb_01)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. The models run in parallel and are independent of each other."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       model_0  model_1  model_2  model_3  model_4  model_5  model_6  model_7  \\\n",
      "0          6.0      0.0      6.0        6      6.0      6.0      6.0        6   \n",
      "1          0.0      6.0      6.0        0      6.0      6.0      6.0        0   \n",
      "2          0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "3          6.0      0.0      6.0        6      6.0      6.0      6.0        6   \n",
      "4          6.0      0.0      6.0        6      6.0      6.0      6.0        6   \n",
      "...        ...      ...      ...      ...      ...      ...      ...      ...   \n",
      "24889      6.0      6.0      6.0        6      6.0      6.0      6.0        6   \n",
      "24890      6.0      0.0      6.0        6      6.0      6.0      6.0        6   \n",
      "24891      1.0      6.0      6.0        6      6.0      6.0      6.0        6   \n",
      "24892      1.0      0.0      6.0        6      6.0      6.0      6.0        6   \n",
      "24893      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "\n",
      "       model_8  model_9  \n",
      "0          6.0      6.0  \n",
      "1          0.0      6.0  \n",
      "2          0.0      0.0  \n",
      "3          1.0      6.0  \n",
      "4          6.0      6.0  \n",
      "...        ...      ...  \n",
      "24889      6.0      6.0  \n",
      "24890      6.0      6.0  \n",
      "24891      1.0      6.0  \n",
      "24892      1.0      6.0  \n",
      "24893      0.0      0.0  \n",
      "\n",
      "[24894 rows x 10 columns]\n"
     ]
    }
   ],
   "source": [
    "bag_vot_df = pd.DataFrame()\n",
    "for i in range(0,len(bag_comb_pred)):\n",
    "    bag_vot_df[f'model_{i}'] =  bag_comb_pred[i]\n",
    "print(bag_vot_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       model_0  model_1  model_2  model_3  model_4  model_5  model_6  model_7  \\\n",
      "0          6.0      0.0      6.0        6      6.0      6.0      6.0        6   \n",
      "1          0.0      6.0      6.0        0      6.0      6.0      6.0        0   \n",
      "2          0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "3          6.0      0.0      6.0        6      6.0      6.0      6.0        6   \n",
      "4          6.0      0.0      6.0        6      6.0      6.0      6.0        6   \n",
      "...        ...      ...      ...      ...      ...      ...      ...      ...   \n",
      "24889      6.0      6.0      6.0        6      6.0      6.0      6.0        6   \n",
      "24890      6.0      0.0      6.0        6      6.0      6.0      6.0        6   \n",
      "24891      1.0      6.0      6.0        6      6.0      6.0      6.0        6   \n",
      "24892      1.0      0.0      6.0        6      6.0      6.0      6.0        6   \n",
      "24893      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "\n",
      "       model_8  model_9  ensemble  \n",
      "0          6.0      6.0         6  \n",
      "1          0.0      6.0         6  \n",
      "2          0.0      0.0         0  \n",
      "3          1.0      6.0         6  \n",
      "4          6.0      6.0         6  \n",
      "...        ...      ...       ...  \n",
      "24889      6.0      6.0         6  \n",
      "24890      6.0      6.0         6  \n",
      "24891      1.0      6.0         6  \n",
      "24892      1.0      6.0         6  \n",
      "24893      0.0      0.0         0  \n",
      "\n",
      "[24894 rows x 11 columns]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0        6\n",
       "1        6\n",
       "2        0\n",
       "3        6\n",
       "4        6\n",
       "        ..\n",
       "24889    6\n",
       "24890    6\n",
       "24891    6\n",
       "24892    6\n",
       "24893    0\n",
       "Name: ensemble, Length: 24894, dtype: int64"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Voting start\n",
    "from scipy.stats import mode\n",
    "# bag_comb_pred_df = pd.DataFrame(bag_comb_pred)\n",
    "# Extract predictions columns\n",
    "\n",
    "# predictions = df[['dnn', 'rf', 'lgbm', 'ada', 'knn', 'mlp', 'svm','cat','xgb']]\n",
    "    # selected_columns = df.loc[:, ~df.columns.isin(['rf'])]\n",
    "predictions = bag_vot_df \n",
    "\n",
    "# predictions = bag_comb_pred_df.loc[:, ~df.columns.isin(['label'])] #df[column_features]\n",
    "\n",
    "# Use the mode function along axis 1 to get the most common prediction for each row\n",
    "ensemble_predictions, _ = mode(predictions.values, axis=1)\n",
    "\n",
    "# Add the ensemble predictions to the DataFrame\n",
    "bag_vot_df['ensemble'] = ensemble_predictions.astype(int)\n",
    "\n",
    "# Display the DataFrame with ensemble predictions\n",
    "print(bag_vot_df)\n",
    "\n",
    "pred_label = bag_vot_df ['ensemble'].values\n",
    "bag_vot_df.pop('ensemble')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0       1      2      3      4      5        6\n",
      "0  2334.0     3.0    0.0    0.0    0.0    0.0      0.0\n",
      "1     0.0  5477.0    0.0    0.0    0.0    0.0      4.0\n",
      "2     0.0     0.0  190.0    0.0    0.0    0.0      0.0\n",
      "3     0.0     0.0    0.0  189.0    0.0    0.0      0.0\n",
      "4     0.0     0.0    4.0    0.0  163.0    0.0      0.0\n",
      "5     0.0     0.0    6.0    0.0    0.0  211.0      0.0\n",
      "6     4.0     0.0    0.0    0.0    0.0    0.0  16309.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9991564232345144\n",
      "Precision total:  0.9924994982208643\n",
      "Recall total:  0.9923056343590659\n",
      "F1 total:  0.9922625590530264\n",
      "BACC total:  0.9923056343590659\n",
      "MCC total:  0.9983560437965728\n"
     ]
    }
   ],
   "source": [
    "name='bag_comb'\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining DNN Model\n",
      "---------------------------------------------------------------------------------\n",
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_6 (Dense)              (None, 3)                 36        \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_6 (Dropout)          (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_7 (Dropout)          (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_8 (Dropout)          (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_10 (Dense)             (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_9 (Dropout)          (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (None, 7)                 28        \n",
      "=================================================================\n",
      "Total params: 112\n",
      "Trainable params: 112\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining DNN Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "start_dnn = time.time()\n",
    "\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "\n",
    "#Model Parameters\n",
    "dropout_rate = 0.2\n",
    "nodes = 3\n",
    "out_layer = 7\n",
    "optimizer='adam'\n",
    "loss='sparse_categorical_crossentropy'\n",
    "epochs=100\n",
    "batch_size=128\n",
    "\n",
    "\n",
    "num_columns = X_train_01.shape[1]\n",
    "\n",
    "dnn_01 = tf.keras.Sequential()\n",
    "\n",
    "# Input layer\n",
    "dnn_01.add(tf.keras.Input(shape=(num_columns,)))\n",
    "\n",
    "# # Dense layers with dropout\n",
    "# dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn_01.add(tf.keras.layers.Dense(2*nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn_01.add(tf.keras.layers.Dense(3*nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn_01.add(tf.keras.layers.Dense(2*nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn.add(tf.keras.layers.Dense(nodes))\n",
    "# dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "\n",
    "\n",
    "# Dense layers with dropout\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# Output layer\n",
    "# dnn_01.add(tf.keras.layers.Dense(out_layer))\n",
    "\n",
    "dnn_01.add(tf.keras.layers.Dense(out_layer, activation='softmax'))\n",
    "# dnn.add(tf.keras.layers.Dense(out_layer, activation='softmax'))\n",
    "\n",
    "\n",
    "dnn_01.compile(optimizer=optimizer, loss=loss,metrics=['accuracy'])\n",
    "\n",
    "dnn_01.summary()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Training DNN\n",
      "---------------------------------------------------------------------------------\n",
      "Epoch 1/100\n",
      "364/364 [==============================] - 1s 3ms/step - loss: 1.3735 - accuracy: 0.6068 - val_loss: 0.9506 - val_accuracy: 0.6534\n",
      "Epoch 2/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.9539 - accuracy: 0.6577 - val_loss: 0.7696 - val_accuracy: 0.6556\n",
      "Epoch 3/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.8409 - accuracy: 0.6811 - val_loss: 0.6906 - val_accuracy: 0.7115\n",
      "Epoch 4/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.7616 - accuracy: 0.7052 - val_loss: 0.6444 - val_accuracy: 0.7374\n",
      "Epoch 5/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.7233 - accuracy: 0.7174 - val_loss: 0.6322 - val_accuracy: 0.7416\n",
      "Epoch 6/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.7039 - accuracy: 0.7231 - val_loss: 0.6439 - val_accuracy: 0.7374\n",
      "Epoch 7/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.6953 - accuracy: 0.7262 - val_loss: 0.6289 - val_accuracy: 0.7419\n",
      "Epoch 8/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.6795 - accuracy: 0.7316 - val_loss: 0.6256 - val_accuracy: 0.7423\n",
      "Epoch 9/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.6683 - accuracy: 0.7344 - val_loss: 0.6249 - val_accuracy: 0.7430\n",
      "Epoch 10/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.6656 - accuracy: 0.7356 - val_loss: 0.6233 - val_accuracy: 0.7428\n",
      "Epoch 11/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.6580 - accuracy: 0.7375 - val_loss: 0.6238 - val_accuracy: 0.7430\n",
      "Epoch 12/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.6536 - accuracy: 0.7395 - val_loss: 0.6229 - val_accuracy: 0.7424\n",
      "Epoch 13/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.6537 - accuracy: 0.7386 - val_loss: 0.6215 - val_accuracy: 0.7426\n",
      "Epoch 14/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.6499 - accuracy: 0.7399 - val_loss: 0.6217 - val_accuracy: 0.7430\n",
      "Epoch 15/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.6510 - accuracy: 0.7402 - val_loss: 0.6217 - val_accuracy: 0.7424\n",
      "Epoch 16/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.6482 - accuracy: 0.7440 - val_loss: 0.6197 - val_accuracy: 0.7429\n",
      "Epoch 17/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.6499 - accuracy: 0.7438 - val_loss: 0.6195 - val_accuracy: 0.7429\n",
      "Epoch 18/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.6485 - accuracy: 0.7447 - val_loss: 0.6194 - val_accuracy: 0.7430\n",
      "Epoch 19/100\n",
      "364/364 [==============================] - 1s 2ms/step - loss: 0.6457 - accuracy: 0.7448 - val_loss: 0.6216 - val_accuracy: 0.7430\n"
     ]
    }
   ],
   "source": [
    "#DNN\n",
    "try:\n",
    "    from keras.callbacks import EarlyStopping\n",
    "\n",
    "    # Define EarlyStopping callback\n",
    "    early_stopping = EarlyStopping(monitor='val_accuracy', patience=10, restore_best_weights=True)\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training DNN')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training DNN', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    # Convert Y_test back to its original format\n",
    "    # y_test = np.argmax(Y_test, axis=1)\n",
    "\n",
    "    # Start the timer\n",
    "    start = time.time()\n",
    "    # dnn_01.fit(X_train_01, y_train_01, epochs=epochs, batch_size=batch_size)\n",
    "    dnn_01.fit(X_train_01, y_train_01, epochs=epochs, batch_size=batch_size,validation_split=0.2, callbacks=[early_stopping])\n",
    "\n",
    "    # model.fit(x_train, Y_train, epochs=100, batch_size=128, validation_split=0.2, callbacks=[early_stopping])\n",
    "\n",
    "    # End the timer\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    # joblib.dump(dnn_01, 'dnn_level_01.joblib')\n",
    "    # dnn_01.save(\"dnn_level_01.h5\")\n",
    "\n",
    "    # Calculate the time taken and print it out\n",
    "    # print(f'Time taken for training: {time_taken} seconds')\n",
    "except: \n",
    "    None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dnn_01 = load_model(\"dnn_level_01.h5\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#DNN\n",
    "try:\n",
    "    start = time.time()\n",
    "    pred_dnn = dnn_01.predict(X_test_01)\n",
    "    preds_dnn_01 = np.argmax(pred_dnn,axis = 1)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "except:\n",
    "        with open(output_file_name, \"a\") as f: print('error', file = f)\n",
    "        preds_dnn_01 = 0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0      1.0  2.0  3.0  4.0  5.0  6.0\n",
      "0.0  2302.0     35.0  0.0  0.0  0.0  0.0  0.0\n",
      "1.0    82.0   5399.0  0.0  0.0  0.0  0.0  0.0\n",
      "2.0   189.0      1.0  0.0  0.0  0.0  0.0  0.0\n",
      "3.0   189.0      0.0  0.0  0.0  0.0  0.0  0.0\n",
      "4.0   162.0      5.0  0.0  0.0  0.0  0.0  0.0\n",
      "5.0   214.0      3.0  0.0  0.0  0.0  0.0  0.0\n",
      "6.0    14.0  16299.0  0.0  0.0  0.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.7472081626094641\n",
      "Precision total:  0.21142642782465262\n",
      "Recall total:  0.2834521890290434\n",
      "F1 total:  0.24219581887823377\n",
      "BACC total:  0.2834521890290434\n",
      "MCC total:  0.4838692608844429\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    name = 'dnn'\n",
    "    pred_label = preds_dnn_01\n",
    "        \n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start_dnn\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n",
    "except: None    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining SVM Model\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#SVM\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining SVM Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "start_svm = time.time()\n",
    "\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "# Instantiate the SGDClassifier with additional hyperparameters\n",
    "clf = SGDClassifier(\n",
    "    loss='hinge',           # hinge loss for linear SVM\n",
    "    penalty='l2',           # L2 regularization to prevent overfitting\n",
    "    alpha=1e-4,             # Learning rate (small value for fine-grained updates)\n",
    "    max_iter=1000,          # Number of passes over the training data\n",
    "    random_state=42,        # Seed for reproducible results\n",
    "    learning_rate='optimal' # Automatically adjusts the learning rate based on the training data\n",
    ")\n",
    "\n",
    "#SVM\n",
    "start = time.time()\n",
    "clf.fit(X_train_01, y_train_01)\n",
    "end = time.time()\n",
    "clf.score(X_train_01, y_train_01)\n",
    "time_taken = end - start\n",
    "with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "joblib.dump(clf, 'svm_level_01.joblib')\n",
    "\n",
    "\n",
    "clf = loaded_model = joblib.load('svm_level_01.joblib')\n",
    "\n",
    "\n",
    "#SVM\n",
    "start = time.time()\n",
    "preds_svm_01 = clf.predict(X_test_01)\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0    2.0    3.0    4.0      5.0  6.0\n",
      "0.0  2301.0    19.0    0.0    0.0    0.0     17.0  0.0\n",
      "1.0     8.0  1239.0   13.0    6.0   40.0   4175.0  0.0\n",
      "2.0     2.0     0.0  176.0   12.0    0.0      0.0  0.0\n",
      "3.0     0.0   187.0    0.0    2.0    0.0      0.0  0.0\n",
      "4.0     0.0     0.0   24.0  138.0    0.0      5.0  0.0\n",
      "5.0    73.0     1.0   19.0    5.0  118.0      1.0  0.0\n",
      "6.0    18.0   496.0    0.0    0.0    0.0  15799.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.7942074395436651\n",
      "Precision total:  0.6768720252119996\n",
      "Recall total:  0.6393689370314541\n",
      "F1 total:  0.6392810029451725\n",
      "BACC total:  0.6393689370314541\n",
      "MCC total:  0.5787190281875111\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "pred_label = preds_svm_01\n",
    "name = 'svm'\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start_svm\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining RF Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training RF\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Prediction RF\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0    2.0    3.0    4.0    5.0      6.0\n",
      "0.0  2324.0     6.0    4.0    0.0    0.0    0.0      3.0\n",
      "1.0     0.0  5468.0    0.0    0.0    0.0    0.0     13.0\n",
      "2.0     0.0     0.0  190.0    0.0    0.0    0.0      0.0\n",
      "3.0     0.0     0.0    0.0  188.0    0.0    0.0      1.0\n",
      "4.0     0.0     0.0   22.0    2.0  143.0    0.0      0.0\n",
      "5.0     0.0     0.0   11.0    0.0    0.0  206.0      0.0\n",
      "6.0     0.0     0.0    0.0    0.0    0.0    0.0  16313.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.997509440025709\n",
      "Precision total:  0.9749058528721897\n",
      "Recall total:  0.9703386654936009\n",
      "F1 total:  0.9706981540818796\n",
      "BACC total:  0.9703386654936009\n",
      "MCC total:  0.9951467024686499\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining RF Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "start_rf = time.time()\n",
    "\n",
    "#Random Forest\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "rf = RandomForestClassifier(max_depth = 5,  n_estimators = 10, min_samples_split = 2, n_jobs = -1)\n",
    "#------------------------------------------------------------------------------\n",
    "\n",
    "if True == True:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training RF')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Training RF', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #RF\n",
    "    start = time.time()\n",
    "    model_rf_01 = rf.fit(X_train_01,y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(model_rf_01, X_train_01, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(model_rf_01, 'rf_base_model_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "    model_rf_01  = joblib.load('rf_base_model_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Prediction RF')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction RF', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #RF\n",
    "    start = time.time()\n",
    "    preds_rf_01 = model_rf_01.predict(X_test_01)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('-------------------------------------------------------', file = f)\n",
    "pred_label = preds_rf_01\n",
    "name='rf'\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start_rf\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining LGBM Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training LGBM\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction LGBM\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0    2.0    3.0    4.0    5.0      6.0\n",
      "0.0  2337.0     0.0    0.0    0.0    0.0    0.0      0.0\n",
      "1.0     0.0  5481.0    0.0    0.0    0.0    0.0      0.0\n",
      "2.0     0.0     0.0  190.0    0.0    0.0    0.0      0.0\n",
      "3.0     0.0     0.0    0.0  189.0    0.0    0.0      0.0\n",
      "4.0     0.0     0.0    0.0    0.0  167.0    0.0      0.0\n",
      "5.0     0.0     0.0    0.0    0.0    0.0  217.0      0.0\n",
      "6.0     0.0     0.0    0.0    0.0    0.0    0.0  16313.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  1.0\n",
      "Precision total:  1.0\n",
      "Recall total:  1.0\n",
      "F1 total:  1.0\n",
      "BACC total:  1.0\n",
      "MCC total:  1.0\n"
     ]
    }
   ],
   "source": [
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining LGBM Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "#LGBM\n",
    "from lightgbm import LGBMClassifier\n",
    "lgbm = LGBMClassifier()\n",
    "\n",
    "start_lgbm = time.time()\n",
    "\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training LGBM')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training LGBM', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    start = time.time()\n",
    "    lgbm.fit(X_train_01, y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(lgbm, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(lgbm, 'lgbm_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "    lgbm = joblib.load('lgbm_01.joblib')\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    print('Prediction LGBM')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction LGBM', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #LGBM\n",
    "    start = time.time()\n",
    "    preds_lgbm_01 = lgbm.predict(X_test_01)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    pred_label = preds_lgbm_01\n",
    "    name='lgbm'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start_lgbm\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining MLP Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training MLP\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0       1      2      3      4      5        6\n",
      "0  2330.0     3.0    1.0    0.0    0.0    0.0      3.0\n",
      "1     3.0  5412.0    1.0    0.0    0.0    0.0     65.0\n",
      "2     0.0     0.0  184.0    0.0    4.0    2.0      0.0\n",
      "3     0.0     0.0    0.0  189.0    0.0    0.0      0.0\n",
      "4     0.0     1.0    5.0    0.0  161.0    0.0      0.0\n",
      "5     0.0     0.0    6.0    0.0    7.0  204.0      0.0\n",
      "6     4.0     4.0    0.0    0.0    0.0    0.0  16305.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9956214348839078\n",
      "Precision total:  0.9788176341661271\n",
      "Recall total:  0.9795014902370064\n",
      "F1 total:  0.9789874124958743\n",
      "BACC total:  0.9795014902370064\n",
      "MCC total:  0.9914622258229167\n"
     ]
    }
   ],
   "source": [
    "\n",
    "#MLP\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining MLP Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "start_mlp = time.time()\n",
    "\n",
    "\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "import time\n",
    "\n",
    "# create MLPClassifier instance\n",
    "mlp = MLPClassifier(hidden_layer_sizes=(100,), max_iter=200, random_state=1)\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training MLP')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Training MLP', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    start = time.time()\n",
    "    MLP = mlp.fit(X_train_01, y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(MLP, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(MLP, 'mlp_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "    MLP = joblib.load('mlp_01.joblib')\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    #MLP\n",
    "    start = time.time()\n",
    "    y_pred = MLP.predict_proba(X_test_01)\n",
    "    preds_mlp_01 = np.argmax(y_pred,axis = 1)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "#MLP\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('MLP 01 model', file = f)\n",
    "    pred_label = preds_mlp_01\n",
    "    name='mlp'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start_mlp\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ADA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining ADA Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training ADA\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction ADA\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0    1.0    2.0    3.0      4.0  5.0  6.0\n",
      "0.0  2335.0    2.0    0.0    0.0      0.0  0.0  0.0\n",
      "1.0  2050.0  200.0    0.0    0.0   3231.0  0.0  0.0\n",
      "2.0     0.0    0.0    0.0  190.0      0.0  0.0  0.0\n",
      "3.0     0.0    0.0  189.0    0.0      0.0  0.0  0.0\n",
      "4.0     0.0    2.0    0.0  165.0      0.0  0.0  0.0\n",
      "5.0     0.0    1.0    0.0  216.0      0.0  0.0  0.0\n",
      "6.0  4326.0    8.0    0.0    0.0  11979.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.5972523499638467\n",
      "Precision total:  0.4690799591582862\n",
      "Recall total:  0.5368543093355195\n",
      "F1 total:  0.38573161344304757\n",
      "BACC total:  0.5368543093355195\n",
      "MCC total:  0.3185629635501104\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining ADA Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "#ADA\n",
    "# from sklearn.multioutput import MultiOutputClassifier\n",
    "start_ada = time.time()\n",
    "\n",
    "\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "import time\n",
    "abc = AdaBoostClassifier(n_estimators=50, learning_rate=1.0)\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training ADA')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training ADA', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #ADA\n",
    "\n",
    "\n",
    "    start = time.time()\n",
    "    ada = abc.fit(X_train_01, y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(ada, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "\n",
    "    # Assuming 'model' is your trained model\n",
    "    joblib.dump(ada, 'ada_01.joblib')\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "    ada = joblib.load('ada_01.joblib')\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    print('Prediction ADA')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction ADA', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #ADA\n",
    "    start = time.time()\n",
    "    preds_ada_01 = ada.predict(X_test_01)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('ADA 01 model', file = f)\n",
    "\n",
    "\n",
    "    pred_label = preds_ada_01\n",
    "    name='ada'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start_ada\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining KNN Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training KNN\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0    2.0    3.0    4.0    5.0      6.0\n",
      "0.0  2328.0     5.0    0.0    0.0    3.0    0.0      1.0\n",
      "1.0     6.0  5428.0    6.0    0.0    0.0    0.0     41.0\n",
      "2.0     1.0     2.0  182.0    0.0    4.0    1.0      0.0\n",
      "3.0     1.0     0.0    2.0  186.0    0.0    0.0      0.0\n",
      "4.0     1.0     0.0    5.0    0.0  161.0    0.0      0.0\n",
      "5.0     0.0     0.0    4.0    0.0    3.0  210.0      0.0\n",
      "6.0     3.0     8.0    0.0    0.0    0.0    0.0  16302.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9961034787498996\n",
      "Precision total:  0.9772714581413472\n",
      "Recall total:  0.9799486206391416\n",
      "F1 total:  0.9784808525555491\n",
      "BACC total:  0.9799486206391416\n",
      "MCC total:  0.9924024571115692\n"
     ]
    }
   ],
   "source": [
    "#KNN\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining KNN Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "start_knn = time.time()\n",
    "\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "knn_clf_01=KNeighborsClassifier(n_neighbors = 5)\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "\n",
    "    #KNN\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training KNN')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training KNN', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    start = time.time()\n",
    "    knn_clf_01.fit(X_train_01,y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(knn_clf, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(knn_clf_01, 'knn_01.joblib')\n",
    "\n",
    "\n",
    "if load_model_knn == 1:\n",
    "    knn_clf_01 = joblib.load('knn_01.joblib')\n",
    "\n",
    "if use_model_knn == 1:\n",
    "\n",
    "    #KNN\n",
    "    start = time.time()\n",
    "    preds_knn =knn_clf_01.predict(X_test_01)\n",
    "    preds_knn\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('KNN 01 model', file = f)\n",
    "\n",
    "    pred_label = preds_knn\n",
    "    name='knn'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "\n",
    "    end = time.time()\n",
    "    time_taken = end - start_knn\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Log Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining Logistic Regression Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training LR \n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0    2.0    3.0    4.0    5.0      6.0\n",
      "0.0  2291.0    26.0    3.0    0.0    0.0    1.0     16.0\n",
      "1.0    14.0  3478.0   14.0    0.0    7.0   20.0   1948.0\n",
      "2.0     1.0     0.0  173.0    1.0    9.0    6.0      0.0\n",
      "3.0     0.0    68.0    3.0  116.0    2.0    0.0      0.0\n",
      "4.0     0.0     1.0   28.0    0.0  137.0    0.0      1.0\n",
      "5.0    54.0     0.0   17.0    2.0    6.0  137.0      1.0\n",
      "6.0    18.0   986.0    0.0    0.0    0.0    0.0  15309.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.8693259419940548\n",
      "Precision total:  0.8572104795547073\n",
      "Recall total:  0.7899007133623164\n",
      "F1 total:  0.8131971896442146\n",
      "BACC total:  0.7899007133623164\n",
      "MCC total:  0.7390967481980706\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#Logistic Regression\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining Logistic Regression Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "logreg_01 = LogisticRegression()\n",
    "start_lr = time.time()\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "\n",
    "    #KNN\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training LR ')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training LR', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    start = time.time()\n",
    "    logreg_01.fit(X_train_01,y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(knn_clf, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(logreg_01, 'logreg_01.joblib')\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "    logreg_01 = joblib.load('logreg_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    #lR\n",
    "    start = time.time()\n",
    "    preds_logreg =logreg_01.predict(X_test_01)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "#LR\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('LR 01 model', file = f)\n",
    "\n",
    "    pred_label = preds_logreg\n",
    "    # pred_label = label[ypred]\n",
    "    name='lr'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start_lr\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Catboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 1.4087270\ttest: 1.4086405\tbest: 1.4086405 (0)\ttotal: 80.7ms\tremaining: 7.99s\n",
      "10:\tlearn: 0.3232701\ttest: 0.3234187\tbest: 0.3234187 (10)\ttotal: 918ms\tremaining: 7.43s\n",
      "20:\tlearn: 0.1141186\ttest: 0.1141954\tbest: 0.1141954 (20)\ttotal: 1.61s\tremaining: 6.06s\n",
      "30:\tlearn: 0.0445193\ttest: 0.0446882\tbest: 0.0446882 (30)\ttotal: 2.08s\tremaining: 4.62s\n",
      "40:\tlearn: 0.0191904\ttest: 0.0194225\tbest: 0.0194225 (40)\ttotal: 2.81s\tremaining: 4.04s\n",
      "50:\tlearn: 0.0097890\ttest: 0.0100773\tbest: 0.0100773 (50)\ttotal: 3.52s\tremaining: 3.38s\n",
      "60:\tlearn: 0.0056518\ttest: 0.0059440\tbest: 0.0059440 (60)\ttotal: 4.3s\tremaining: 2.75s\n",
      "70:\tlearn: 0.0035469\ttest: 0.0038665\tbest: 0.0038665 (70)\ttotal: 5.03s\tremaining: 2.06s\n",
      "80:\tlearn: 0.0026485\ttest: 0.0029522\tbest: 0.0029522 (80)\ttotal: 5.94s\tremaining: 1.39s\n",
      "90:\tlearn: 0.0020164\ttest: 0.0023158\tbest: 0.0023158 (90)\ttotal: 6.79s\tremaining: 672ms\n",
      "99:\tlearn: 0.0016471\ttest: 0.0019452\tbest: 0.0019452 (99)\ttotal: 7.54s\tremaining: 0us\n",
      "\n",
      "bestTest = 0.001945245421\n",
      "bestIteration = 99\n",
      "\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0    2.0    3.0    4.0    5.0      6.0\n",
      "0.0  2337.0     0.0    0.0    0.0    0.0    0.0      0.0\n",
      "1.0     0.0  5480.0    0.0    0.0    0.0    0.0      1.0\n",
      "2.0     0.0     0.0  190.0    0.0    0.0    0.0      0.0\n",
      "3.0     0.0     0.0    0.0  189.0    0.0    0.0      0.0\n",
      "4.0     2.0     0.0    0.0    0.0  165.0    0.0      0.0\n",
      "5.0     0.0     0.0    1.0    0.0    0.0  216.0      0.0\n",
      "6.0     0.0     0.0    0.0    0.0    0.0    0.0  16313.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.999839318711336\n",
      "Precision total:  0.9991211477989488\n",
      "Recall total:  0.997604744100946\n",
      "F1 total:  0.9983560235522603\n",
      "BACC total:  0.997604744100946\n",
      "MCC total:  0.999686821339456\n"
     ]
    }
   ],
   "source": [
    "import catboost\n",
    "start = time.time()\n",
    "\n",
    "cat_01 = catboost.CatBoostClassifier(iterations=100, depth=6, learning_rate=0.1, loss_function='MultiClass', custom_metric='Accuracy')\n",
    "\n",
    "# Fit the model\n",
    "cat_01.fit(X_train_01, y_train_01, eval_set=(X_test_01, y_test_01), verbose=10)\n",
    "\n",
    "# Make predictions on the test set\n",
    "preds_cat = cat_01.predict(X_test_01)\n",
    "preds_cat = np.squeeze(preds_cat)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('catboost', file = f)\n",
    "\n",
    "\n",
    "pred_label = preds_cat\n",
    "name='cat'\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0    2.0    3.0    4.0    5.0      6.0\n",
      "0.0  2337.0     0.0    0.0    0.0    0.0    0.0      0.0\n",
      "1.0     0.0  5481.0    0.0    0.0    0.0    0.0      0.0\n",
      "2.0     0.0     0.0  190.0    0.0    0.0    0.0      0.0\n",
      "3.0     0.0     0.0    0.0  189.0    0.0    0.0      0.0\n",
      "4.0     0.0     0.0    0.0    0.0  167.0    0.0      0.0\n",
      "5.0     0.0     0.0    0.0    0.0    0.0  217.0      0.0\n",
      "6.0     0.0     0.0    0.0    0.0    0.0    0.0  16313.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  1.0\n",
      "Precision total:  1.0\n",
      "Recall total:  1.0\n",
      "F1 total:  1.0\n",
      "BACC total:  1.0\n",
      "MCC total:  1.0\n"
     ]
    }
   ],
   "source": [
    "\n",
    "import xgboost as xgb\n",
    "start = time.time()\n",
    "\n",
    "# Create a DMatrix for XGBoost\n",
    "dtrain = xgb.DMatrix(X_train_01, label=y_train_01)\n",
    "dtest = xgb.DMatrix(X_test_01, label=y_test_01)\n",
    "\n",
    "# Set XGBoost parameters\n",
    "params = {\n",
    "    'objective': 'multi:softmax',  # for multi-class classification\n",
    "    'num_class': 7,  # specify the number of classes\n",
    "    'max_depth': 3,\n",
    "    'learning_rate': 0.1,\n",
    "    'eval_metric': 'mlogloss'  # metric for multi-class classification\n",
    "}\n",
    "\n",
    "# Train the XGBoost model\n",
    "num_round = 100\n",
    "xgb_01 = xgb.train(params, dtrain, num_round)\n",
    "\n",
    "# Make predictions on the test set\n",
    "preds_xgb_01 = xgb_01.predict(dtest)\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('xgboost base model', file = f)\n",
    "\n",
    "    pred_label = preds_xgb_01\n",
    "    name='xgb'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ### Generating Summary Metric Table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+----------+-----------+----------+-----------+\n",
      "| Models      |   ACC-01 |    PRE-01 |   REC-01 |     F1-01 |\n",
      "+=============+==========+===========+==========+===========+\n",
      "| XGB         | 1        | 1         | 1        | 1         |\n",
      "+-------------+----------+-----------+----------+-----------+\n",
      "| LGBM        | 1        | 1         | 1        | 1         |\n",
      "+-------------+----------+-----------+----------+-----------+\n",
      "| Bag_DT      | 1        | 1         | 1        | 1         |\n",
      "+-------------+----------+-----------+----------+-----------+\n",
      "| Bag_lgbm    | 1        | 1         | 1        | 1         |\n",
      "+-------------+----------+-----------+----------+-----------+\n",
      "| CAT         | 0.999839 | 0.999121  | 0.997605 | 0.998356  |\n",
      "+-------------+----------+-----------+----------+-----------+\n",
      "| Bag_cat     | 0.999759 | 0.997649  | 0.996288 | 0.996948  |\n",
      "+-------------+----------+-----------+----------+-----------+\n",
      "| Bag_comb    | 0.999156 | 0.992499  | 0.992306 | 0.992263  |\n",
      "+-------------+----------+-----------+----------+-----------+\n",
      "| Bag_knn     | 0.996144 | 0.977699  | 0.980752 | 0.979085  |\n",
      "+-------------+----------+-----------+----------+-----------+\n",
      "| MLP         | 0.995621 | 0.978818  | 0.979501 | 0.978987  |\n",
      "+-------------+----------+-----------+----------+-----------+\n",
      "| KNN         | 0.996103 | 0.977271  | 0.979949 | 0.978481  |\n",
      "+-------------+----------+-----------+----------+-----------+\n",
      "| Bag_mlp     | 0.996103 | 0.977926  | 0.977877 | 0.977687  |\n",
      "+-------------+----------+-----------+----------+-----------+\n",
      "| Bag_rf      | 0.997831 | 0.977548  | 0.978242 | 0.976819  |\n",
      "+-------------+----------+-----------+----------+-----------+\n",
      "| RF          | 0.997509 | 0.974906  | 0.970339 | 0.970698  |\n",
      "+-------------+----------+-----------+----------+-----------+\n",
      "| Bag_LR      | 0.872941 | 0.868967  | 0.849074 | 0.854708  |\n",
      "+-------------+----------+-----------+----------+-----------+\n",
      "| LR          | 0.869326 | 0.85721   | 0.789901 | 0.813197  |\n",
      "+-------------+----------+-----------+----------+-----------+\n",
      "| Bag_ada     | 0.666185 | 0.826385  | 0.785914 | 0.79262   |\n",
      "+-------------+----------+-----------+----------+-----------+\n",
      "| Bag_svm     | 0.814172 | 0.768391  | 0.656643 | 0.660106  |\n",
      "+-------------+----------+-----------+----------+-----------+\n",
      "| SVM         | 0.794207 | 0.676872  | 0.639369 | 0.639281  |\n",
      "+-------------+----------+-----------+----------+-----------+\n",
      "| ADA         | 0.597252 | 0.46908   | 0.536854 | 0.385732  |\n",
      "+-------------+----------+-----------+----------+-----------+\n",
      "| DNN         | 0.747208 | 0.211426  | 0.283452 | 0.242196  |\n",
      "+-------------+----------+-----------+----------+-----------+\n",
      "| avg         | 0.219933 | 0.0397056 | 0.143122 | 0.0530925 |\n",
      "+-------------+----------+-----------+----------+-----------+\n",
      "| weighed_avg | 0.220254 | 0.0451156 | 0.14312  | 0.0523393 |\n",
      "+-------------+----------+-----------+----------+-----------+\n",
      "| VOTING      | 0        | 0         | 0        | 0         |\n",
      "+-------------+----------+-----------+----------+-----------+\n"
     ]
    }
   ],
   "source": [
    "from tabulate import tabulate\n",
    "\n",
    "# Assuming data is a 110x4 list, where each row is a sublist\n",
    "# data =  [[\"Row {} Col {}\".format(i + 1, j + 1) for j in range(4)] for i in range(110)]\n",
    "names_models = ['ADA',\n",
    "                'SVM',\n",
    "                'DNN',\n",
    "                'MLP',\n",
    "                'KNN',\n",
    "                'CAT',\n",
    "                'XGB',\n",
    "                'LGBM',\n",
    "                'RF',\n",
    "                'LR',\n",
    "                'VOTING',\n",
    "                'Bag_svm',\n",
    "                'Bag_knn',\n",
    "                'Bag_DT',\n",
    "                'Bag_LR',\n",
    "                'Bag_mlp',\n",
    "\n",
    "                'Bag_rf',\n",
    "                'Bag_ada',\n",
    "                'Bag_lgbm',\n",
    "                # 'Bag_xgb',\n",
    "                'Bag_cat',\n",
    "                'Bag_comb',\n",
    "                'avg',\n",
    "                'weighed_avg'\n",
    "                ]\n",
    "\n",
    "data = [[\"\" for _ in range(5)] for _ in range(len(names_models))]\n",
    "\n",
    "level_01_acc = [\n",
    "                ada_acc_01,\n",
    "                svm_acc_01,\n",
    "                dnn_acc_01,\n",
    "                mlp_acc_01,\n",
    "                knn_acc_01,\n",
    "                cat_acc_01,\n",
    "                xgb_acc_01,\n",
    "                lgbm_acc_01,\n",
    "                rf_acc_01,\n",
    "                lr_acc_01,\n",
    "                voting_acc_01,\n",
    "                bag_svm_acc_01,\n",
    "                bag_knn_acc_01,\n",
    "                bag_dt_acc_01,\n",
    "                bag_lr_acc_01,\n",
    "                bag_mlp_acc_01,\n",
    "\n",
    "                bag_rf_acc_01,\n",
    "                bag_ada_acc_01,\n",
    "                bag_lgbm_acc_01,\n",
    "                # bag_xgb_acc_01,\n",
    "                bag_cat_acc_01,\n",
    "                bag_comb_acc_01,\n",
    "\n",
    "                avg_acc_01,\n",
    "                weighed_avg_acc_01\n",
    "                ]  \n",
    "\n",
    "\n",
    "level_01_pre = [\n",
    "                ada_pre_01,\n",
    "                svm_pre_01,\n",
    "                dnn_pre_01,\n",
    "                mlp_pre_01,\n",
    "                knn_pre_01,\n",
    "                cat_pre_01,\n",
    "                xgb_pre_01,\n",
    "                lgbm_pre_01,\n",
    "                rf_pre_01,\n",
    "                lr_pre_01,\n",
    "                voting_pre_01,\n",
    "                bag_svm_pre_01,\n",
    "                bag_knn_pre_01,\n",
    "                bag_dt_pre_01,\n",
    "                bag_lr_pre_01,\n",
    "                bag_mlp_pre_01,\n",
    "\n",
    "                bag_rf_pre_01,\n",
    "                bag_ada_pre_01,\n",
    "                bag_lgbm_pre_01,\n",
    "                # bag_xgb_pre_01,\n",
    "                bag_cat_pre_01,\n",
    "                bag_comb_pre_01,\n",
    "\n",
    "                avg_pre_01,\n",
    "                weighed_avg_pre_01\n",
    "                ]  \n",
    "\n",
    "level_01_rec = [\n",
    "                ada_rec_01,\n",
    "                svm_rec_01,\n",
    "                dnn_rec_01,\n",
    "                mlp_rec_01,\n",
    "                knn_rec_01,\n",
    "                cat_rec_01,\n",
    "                xgb_rec_01,\n",
    "                lgbm_rec_01,\n",
    "                rf_rec_01,\n",
    "                lr_rec_01,\n",
    "                voting_rec_01,\n",
    "                bag_svm_rec_01,\n",
    "                bag_knn_rec_01,\n",
    "                bag_dt_rec_01,\n",
    "                bag_lr_rec_01,\n",
    "                bag_mlp_rec_01,\n",
    "\n",
    "                bag_rf_rec_01,\n",
    "                bag_ada_rec_01,\n",
    "                bag_lgbm_rec_01,\n",
    "                # bag_xgb_rec_01,\n",
    "                bag_cat_rec_01,\n",
    "                bag_comb_rec_01,\n",
    "\n",
    "                avg_rec_01,\n",
    "                weighed_avg_rec_01\n",
    "                ]  \n",
    "\n",
    "level_01_f1 = [\n",
    "                ada_f1_01,\n",
    "                svm_f1_01,\n",
    "                dnn_f1_01,\n",
    "                mlp_f1_01,\n",
    "                knn_f1_01,\n",
    "                cat_f1_01,\n",
    "                xgb_f1_01,\n",
    "                lgbm_f1_01,\n",
    "                rf_f1_01,\n",
    "                lr_f1_01,\n",
    "                voting_f1_01,\n",
    "                bag_svm_f1_01,\n",
    "                bag_knn_f1_01,\n",
    "                bag_dt_f1_01,\n",
    "                bag_lr_f1_01,\n",
    "                bag_mlp_f1_01,\n",
    "\n",
    "                bag_rf_f1_01,\n",
    "                bag_ada_f1_01,\n",
    "                bag_lgbm_f1_01,\n",
    "                # bag_xgb_f1_01,\n",
    "                bag_cat_f1_01,\n",
    "                bag_comb_f1_01,\n",
    "\n",
    "                avg_f1_01,\n",
    "                weighed_avg_f1_01\n",
    "                ]  \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# Combine data into a list of tuples for sorting\n",
    "model_data = list(zip(names_models, level_01_acc, level_01_pre, level_01_rec, level_01_f1))\n",
    "\n",
    "# Sort by F1-01 score in descending order\n",
    "model_data_sorted = sorted(model_data, key=lambda x: x[4], reverse=True)\n",
    "\n",
    "# Separate the sorted data back into individual lists\n",
    "sorted_names_models, sorted_level_01_acc, sorted_level_01_pre, sorted_level_01_rec, sorted_level_01_f1 = zip(*model_data_sorted)\n",
    "\n",
    "# Assign the sorted data to the table\n",
    "for i in range(len(sorted_names_models)):\n",
    "    data[i][0] = sorted_names_models[i]\n",
    "    data[i][1] = sorted_level_01_acc[i]\n",
    "    data[i][2] = sorted_level_01_pre[i] \n",
    "    data[i][3] = sorted_level_01_rec[i] \n",
    "    data[i][4] = sorted_level_01_f1[i]\n",
    "\n",
    "# Define column headers\n",
    "headers = [\"Models\", \"ACC-01\", \"PRE-01\", \"REC-01\", \"F1-01\"]\n",
    "\n",
    "# Print the table\n",
    "table = tabulate(data, headers=headers, tablefmt=\"grid\")\n",
    "with open(output_file_name, \"a\") as f: print('Summary table', file = f)\n",
    "if pick_prob == 1: \n",
    "    with open(output_file_name, \"a\") as f: print('Level 01 - Probabilities', file = f)\n",
    "else:\n",
    "    with open(output_file_name, \"a\") as f: print('Level 01 - CLASSES', file = f)\n",
    "if feature_selection_bit == 1: \n",
    "    with open(output_file_name, \"a\") as f: print('Feature Selection was applied', file = f)\n",
    "else:\n",
    "    with open(output_file_name, \"a\") as f: print('All features were used', file = f)\n",
    "\n",
    "\n",
    "    \n",
    "print(table)\n",
    "with open(output_file_name, \"a\") as f: print(table, file = f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+----------------+\n",
      "| Models      |   time-01(sec) |\n",
      "+=============+================+\n",
      "| avg         |      0.0845072 |\n",
      "+-------------+----------------+\n",
      "| weighed_avg |      0.0960176 |\n",
      "+-------------+----------------+\n",
      "| RF          |      0.347909  |\n",
      "+-------------+----------------+\n",
      "| SVM         |      0.521412  |\n",
      "+-------------+----------------+\n",
      "| Bag_DT      |      1.48349   |\n",
      "+-------------+----------------+\n",
      "| Bag_svm     |      4.09211   |\n",
      "+-------------+----------------+\n",
      "| Bag_rf      |      4.20171   |\n",
      "+-------------+----------------+\n",
      "| ADA         |      5.06811   |\n",
      "+-------------+----------------+\n",
      "| CAT         |      8.03967   |\n",
      "+-------------+----------------+\n",
      "| Bag_lgbm    |      8.81799   |\n",
      "+-------------+----------------+\n",
      "| LR          |     11.2648    |\n",
      "+-------------+----------------+\n",
      "| Bag_cat     |     12.0622    |\n",
      "+-------------+----------------+\n",
      "| DNN         |     17.0509    |\n",
      "+-------------+----------------+\n",
      "| Bag_ada     |     17.4516    |\n",
      "+-------------+----------------+\n",
      "| KNN         |     20.3749    |\n",
      "+-------------+----------------+\n",
      "| Bag_LR      |     24.7525    |\n",
      "+-------------+----------------+\n",
      "| Bag_knn     |     83.353     |\n",
      "+-------------+----------------+\n",
      "| MLP         |    144.713     |\n",
      "+-------------+----------------+\n",
      "| Bag_comb    |    197.654     |\n",
      "+-------------+----------------+\n",
      "| XGB         |    326.647     |\n",
      "+-------------+----------------+\n",
      "| Bag_mlp     |    783.752     |\n",
      "+-------------+----------------+\n",
      "| LGBM        |    914.519     |\n",
      "+-------------+----------------+\n",
      "| VOTING      |   9999         |\n",
      "+-------------+----------------+\n"
     ]
    }
   ],
   "source": [
    "# implement time table\n",
    "from tabulate import tabulate\n",
    "\n",
    "names_models = ['ADA',\n",
    "                'SVM',\n",
    "                'DNN',\n",
    "                'MLP',\n",
    "                'KNN',\n",
    "                'CAT',\n",
    "                'XGB',\n",
    "                'LGBM',\n",
    "                'RF',\n",
    "                'LR',\n",
    "                'VOTING',\n",
    "                'Bag_svm',\n",
    "                'Bag_knn',\n",
    "                'Bag_DT',\n",
    "                'Bag_LR',\n",
    "                'Bag_mlp',\n",
    "\n",
    "                'Bag_rf',\n",
    "                'Bag_ada',\n",
    "                'Bag_lgbm',\n",
    "                # 'Bag_xgb',\n",
    "                'Bag_cat',\n",
    "                'Bag_comb',\n",
    "                'avg',\n",
    "                'weighed_avg'\n",
    "                ]\n",
    "\n",
    "data = [[\"\" for _ in range(2)] for _ in range(len(names_models))]\n",
    "\n",
    "level_01_time = [\n",
    "                ada_time_01,\n",
    "                svm_time_01,\n",
    "                dnn_time_01,\n",
    "                mlp_time_01,\n",
    "                knn_time_01,\n",
    "                cat_time_01,\n",
    "                xgb_time_01,\n",
    "                lgbm_time_01,\n",
    "                rf_time_01,\n",
    "                lr_time_01,\n",
    "                voting_time_01,\n",
    "                bag_svm_time_01,\n",
    "                bag_knn_time_01,\n",
    "                bag_dt_time_01,\n",
    "                bag_lr_time_01,\n",
    "                bag_mlp_time_01,\n",
    "\n",
    "                bag_rf_time_01,\n",
    "                bag_ada_time_01,\n",
    "                bag_lgbm_time_01,\n",
    "                # bag_xgb_time_01,\n",
    "                bag_cat_time_01,\n",
    "                bag_comb_time_01,\n",
    "\n",
    "                avg_time_01,\n",
    "                weighed_avg_time_01\n",
    "                ]  \n",
    "\n",
    "\n",
    "# Combine data into a list of tuples for sorting\n",
    "model_data = list(zip(names_models, level_01_time))\n",
    "\n",
    "# Sort by F1-01 score in descending order\n",
    "model_data_sorted = sorted(model_data, key=lambda x: x[1], reverse=False)\n",
    "\n",
    "# Separate the sorted data back into individual lists\n",
    "sorted_names_models, sorted_level_01_time = zip(*model_data_sorted)\n",
    "\n",
    "# Assign the sorted data to the table\n",
    "for i in range(len(sorted_names_models)):\n",
    "    data[i][0] = sorted_names_models[i]\n",
    "    data[i][1] = sorted_level_01_time[i]\n",
    "\n",
    "# Define column headers\n",
    "headers = [\"Models\", \"time-01(sec)\"]\n",
    "\n",
    "\n",
    "# Print the table\n",
    "table = tabulate(data, headers=headers, tablefmt=\"grid\")\n",
    "with open(output_file_name, \"a\") as f: print('Time is counted is seconds', file = f)\n",
    "print(table)\n",
    "with open(output_file_name, \"a\") as f: print(table, file = f)\n",
    "end_program = time.time()\n",
    "time_program = end_program - start_program\n",
    "with open(output_file_name, \"a\") as f: print('Running time of entire program is:', time_program ,' seconds',file = f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ------------------------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if generate_feature_importance == 1:\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('Generating SHAP explanation')\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('')\n",
    "  with open(output_file_name, \"a\") as f:print('ADA FEATURE IMPORTANCE',file = f)\n",
    "\n",
    "      #START TIMER MODEL\n",
    "  start = time.time()\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('Generating explainer')\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('')\n",
    "  test = X_test_01\n",
    "  train = X_train_01\n",
    "  # ## Summary Bar Plot Global\n",
    "  start_index = 0\n",
    "  end_index = 250\n",
    "  # test.pop('Label')\n",
    "  # test.pop('is_train')\n",
    "  # print(label2)\n",
    "\n",
    "\n",
    "  # models = [ada,dnn_01,clf,knn_clf_01,cat_01,xgb_01, rf, lgbm, mlp,logreg_01]\n",
    "  explainer = shap.KernelExplainer(ada.predict_proba, test[start_index:end_index])\n",
    "\n",
    "  shap_values = explainer.shap_values(test[start_index:end_index])\n",
    "\n",
    "  shap.summary_plot(shap_values = shap_values,\n",
    "                    features = test[start_index:end_index],\n",
    "                    # class_names=[column_features[:-1]],\n",
    "                    show=False)\n",
    "\n",
    "  # if feature_selection_bit == 1 # On\n",
    "  # pick_prob = 0 # set equal one to choose the dataset with probabilities, set to 0 to choose one with the classes.\n",
    "  if pick_prob == 1:\n",
    "    plt.savefig('ADA_SHAP_NSL_prob_01.png')\n",
    "  elif pick_prob == 0:\n",
    "    plt.savefig('ADA_SHAP_NSL_class_01.png')\n",
    "        \n",
    "  else: None\n",
    "  plt.clf()\n",
    "\n",
    "\n",
    "  vals= np.abs(shap_values).mean(1)\n",
    "  feature_importance = pd.DataFrame(list(zip(train.columns, sum(vals))), columns=['col_name','feature_importance_vals'])\n",
    "  feature_importance.sort_values(by=['feature_importance_vals'], ascending=False,inplace=True)\n",
    "  feature_importance.head()\n",
    "  print(feature_importance.to_string())\n",
    "\n",
    "  with open(output_file_name, \"a\") as f:print('Feature Importance: ',feature_importance.to_string(),file = f)\n",
    "\n",
    "\n",
    "\n",
    "  end = time.time()\n",
    "  with open(output_file_name, \"a\") as f:print('ELAPSE TIME LIME GLOBAL: ',(end - start)/60, 'min',file = f)\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  # feature_importance_vals = 'feature_importance_vals'  # Replace with the name of the column you want to extract\n",
    "  feature_val = feature_importance['feature_importance_vals'].tolist()\n",
    "\n",
    "  # col_name = 'col_name'  # Replace with the name of the column you want to extract\n",
    "  feature_name = feature_importance['col_name'].tolist()\n",
    "\n",
    "\n",
    "  # for item1, item2 in zip(feature_name, feature_val):\n",
    "  #     print(item1, item2)\n",
    "\n",
    "\n",
    "  # Use zip to combine the two lists, sort based on list1, and then unzip them\n",
    "  zipped_lists = list(zip(feature_name, feature_val))\n",
    "  zipped_lists.sort(key=lambda x: x[1],reverse=True)\n",
    "\n",
    "  # Convert the sorted result back into separate lists\n",
    "  sorted_list1, sorted_list2 = [list(x) for x in zip(*zipped_lists)]\n",
    "\n",
    "  # for k in sorted_list1:\n",
    "  #   with open(output_file_name, \"a\") as f: print(\"df.pop('\",k,\"')\", sep='', file = f)\n",
    "\n",
    "  # with open(output_file_name, \"a\") as f:print(\"Trial_ =[\", file = f)\n",
    "  # for k in sorted_list1:\n",
    "  #   with open(output_file_name, \"a\") as f:print(\"'\",k,\"',\", sep='', file = f)\n",
    "  # with open(output_file_name, \"a\") as f:print(\"]\", file = f)\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# explainer = shap.TreeExplainer(model)\n",
    "# start_index = 0\n",
    "# end_index = samples\n",
    "# shap_values = explainer.shap_values(test[start_index:end_index])\n",
    "# shap_obj = explainer(test[start_index:end_index])\n",
    "# shap.summary_plot(shap_values = shap_values,\n",
    "#                   features = test[start_index:end_index],\n",
    "#                 show=False)\n",
    "# plt.savefig('Light_SHAP_CIC_Summary.png')\n",
    "# plt.clf()\n",
    "\n",
    "\n",
    "# vals= np.abs(shap_values).mean(1)\n",
    "# feature_importance = pd.DataFrame(list(zip(train.columns, sum(vals))), columns=['col_name','feature_importance_vals'])\n",
    "# feature_importance.sort_values(by=['feature_importance_vals'], ascending=False,inplace=True)\n",
    "# feature_importance.head()\n",
    "# print(feature_importance.to_string())\n",
    "# print('---------------------------------------------------------------------------------')\n",
    "# # feature_importance_vals = 'feature_importance_vals'  # Replace with the name of the column you want to extract\n",
    "# feature_val = feature_importance['feature_importance_vals'].tolist()\n",
    "\n",
    "# # col_name = 'col_name'  # Replace with the name of the column you want to extract\n",
    "# feature_name = feature_importance['col_name'].tolist()\n",
    "\n",
    "\n",
    "# # for item1, item2 in zip(feature_name, feature_val):\n",
    "# #     print(item1, item2)\n",
    "\n",
    "\n",
    "# # Use zip to combine the two lists, sort based on list1, and then unzip them\n",
    "# zipped_lists = list(zip(feature_name, feature_val))\n",
    "# zipped_lists.sort(key=lambda x: x[1],reverse=True)\n",
    "\n",
    "# # Convert the sorted result back into separate lists\n",
    "# sorted_list1, sorted_list2 = [list(x) for x in zip(*zipped_lists)]\n",
    "\n",
    "# for k in sorted_list1:\n",
    "#   with open(output_file_name, \"a\") as f:print(\"df.pop('\",k,\"')\", sep='', file = f)\n",
    "\n",
    "# # with open(output_file_name, \"a\") as f:print(\"Trial_ =[\", file = f)\n",
    "# for k in sorted_list1:\n",
    "#   with open(output_file_name, \"a\") as f:print(\"'\",k,\"',\", sep='', file = f)\n",
    "# with open(output_file_name, \"a\") as f:print(\"]\", file = f)\n",
    "# print('---------------------------------------------------------------------------------')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if generate_feature_importance == 1:\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('Generating SHAP explanation')\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('')\n",
    "\n",
    "  with open(output_file_name, \"a\") as f:print('XGB FEATURE IMPORTANCE',file = f)\n",
    "\n",
    "      #START TIMER MODEL\n",
    "  start = time.time()\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('Generating explainer')\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('')\n",
    "  test = X_test_01\n",
    "  train = X_train_01\n",
    "  # ## Summary Bar Plot Global\n",
    "  start_index = 0\n",
    "  end_index = 250\n",
    "  # test.pop('Label')\n",
    "  # test.pop('is_train')\n",
    "  # print(label2)\n",
    "\n",
    "\n",
    "  explainer = shap.TreeExplainer(xgb_01)\n",
    "\n",
    "  shap_values = explainer.shap_values(test[start_index:end_index])\n",
    "  shap_obj = explainer(test[start_index:end_index])\n",
    "  shap.summary_plot(shap_values = shap_values,\n",
    "                    features = test[start_index:end_index],\n",
    "                  show=False)\n",
    "  # plt.clf()\n",
    "\n",
    "  # if feature_selection_bit == 1 # On\n",
    "  # pick_prob = 0 # set equal one to choose the dataset with probabilities, set to 0 to choose one with the classes.\n",
    "  if pick_prob == 1:\n",
    "    plt.savefig('XGB_SHAP_NSL_prob_01.png')\n",
    "  elif pick_prob == 0:\n",
    "    plt.savefig('XGB_SHAP_NSL_class_01.png')\n",
    "\n",
    "  else: None\n",
    "  plt.clf()\n",
    "\n",
    "\n",
    "  vals= np.abs(shap_values).mean(1)\n",
    "  feature_importance = pd.DataFrame(list(zip(train.columns, sum(vals))), columns=['col_name','feature_importance_vals'])\n",
    "  feature_importance.sort_values(by=['feature_importance_vals'], ascending=False,inplace=True)\n",
    "  feature_importance.head()\n",
    "  print(feature_importance.to_string())\n",
    "  with open(output_file_name, \"a\") as f:print('Feature Importance: ',feature_importance.to_string(),file = f)\n",
    "\n",
    "\n",
    "\n",
    "  end = time.time()\n",
    "  with open(output_file_name, \"a\") as f:print('ELAPSE TIME LIME GLOBAL: ',(end - start)/60, 'min',file = f)\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  # feature_importance_vals = 'feature_importance_vals'  # Replace with the name of the column you want to extract\n",
    "  feature_val = feature_importance['feature_importance_vals'].tolist()\n",
    "\n",
    "  # col_name = 'col_name'  # Replace with the name of the column you want to extract\n",
    "  feature_name = feature_importance['col_name'].tolist()\n",
    "\n",
    "\n",
    "  # for item1, item2 in zip(feature_name, feature_val):\n",
    "  #     print(item1, item2)\n",
    "\n",
    "\n",
    "  # Use zip to combine the two lists, sort based on list1, and then unzip them\n",
    "  zipped_lists = list(zip(feature_name, feature_val))\n",
    "  zipped_lists.sort(key=lambda x: x[1],reverse=True)\n",
    "\n",
    "  # Convert the sorted result back into separate lists\n",
    "  sorted_list1, sorted_list2 = [list(x) for x in zip(*zipped_lists)]\n",
    "\n",
    "  # for k in sorted_list1:\n",
    "  #   with open(output_file_name, \"a\") as f: print(\"df.pop('\",k,\"')\", sep='', file = f)\n",
    "\n",
    "  # with open(output_file_name, \"a\") as f:print(\"Trial_ =[\", file = f)\n",
    "  # for k in sorted_list1:\n",
    "  #   with open(output_file_name, \"a\") as f:print(\"'\",k,\"',\", sep='', file = f)\n",
    "  # with open(output_file_name, \"a\") as f:print(\"]\", file = f)\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if generate_feature_importance == 1:\n",
    "\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('Generating SHAP explanation')\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('')\n",
    "\n",
    "  with open(output_file_name, \"a\") as f:print('LGBM FEATURE IMPORTANCE',file = f)\n",
    "\n",
    "      #START TIMER MODEL\n",
    "  start = time.time()\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('Generating explainer')\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('')\n",
    "  test = X_test_01\n",
    "  train = X_train_01\n",
    "  # ## Summary Bar Plot Global\n",
    "  start_index = 0\n",
    "  end_index = 250\n",
    "  # test.pop('Label')\n",
    "  # test.pop('is_train')\n",
    "  # print(label2)\n",
    "\n",
    "\n",
    "  explainer = shap.TreeExplainer(lgbm)\n",
    "\n",
    "  shap_values = explainer.shap_values(test[start_index:end_index])\n",
    "  shap_obj = explainer(test[start_index:end_index])\n",
    "  shap.summary_plot(shap_values = shap_values,\n",
    "                    features = test[start_index:end_index],\n",
    "                  show=False)\n",
    "  # plt.clf()\n",
    "\n",
    "  # if feature_selection_bit == 1 # On\n",
    "  # pick_prob = 0 # set equal one to choose the dataset with probabilities, set to 0 to choose one with the classes.\n",
    "  if pick_prob == 1:\n",
    "    plt.savefig('LGBM_SHAP_NSL_prob_01.png')\n",
    "  elif pick_prob == 0:\n",
    "    plt.savefig('LGBM_SHAP_NSL_class_01.png')\n",
    "\n",
    "  else: None\n",
    "  plt.clf()\n",
    "\n",
    "\n",
    "  vals= np.abs(shap_values).mean(1)\n",
    "  feature_importance = pd.DataFrame(list(zip(train.columns, sum(vals))), columns=['col_name','feature_importance_vals'])\n",
    "  feature_importance.sort_values(by=['feature_importance_vals'], ascending=False,inplace=True)\n",
    "  feature_importance.head()\n",
    "  print(feature_importance.to_string())\n",
    "  with open(output_file_name, \"a\") as f:print('Feature Importance: ',feature_importance.to_string(),file = f)\n",
    "\n",
    "\n",
    "\n",
    "  end = time.time()\n",
    "  with open(output_file_name, \"a\") as f:print('ELAPSE TIME LIME GLOBAL: ',(end - start)/60, 'min',file = f)\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  # feature_importance_vals = 'feature_importance_vals'  # Replace with the name of the column you want to extract\n",
    "  feature_val = feature_importance['feature_importance_vals'].tolist()\n",
    "\n",
    "  # col_name = 'col_name'  # Replace with the name of the column you want to extract\n",
    "  feature_name = feature_importance['col_name'].tolist()\n",
    "\n",
    "\n",
    "  # for item1, item2 in zip(feature_name, feature_val):\n",
    "  #     print(item1, item2)\n",
    "\n",
    "\n",
    "  # Use zip to combine the two lists, sort based on list1, and then unzip them\n",
    "  zipped_lists = list(zip(feature_name, feature_val))\n",
    "  zipped_lists.sort(key=lambda x: x[1],reverse=True)\n",
    "\n",
    "  # Convert the sorted result back into separate lists\n",
    "  sorted_list1, sorted_list2 = [list(x) for x in zip(*zipped_lists)]\n",
    "\n",
    "  # for k in sorted_list1:\n",
    "  #   with open(output_file_name, \"a\") as f: print(\"df.pop('\",k,\"')\", sep='', file = f)\n",
    "\n",
    "  # with open(output_file_name, \"a\") as f:print(\"Trial_ =[\", file = f)\n",
    "  # for k in sorted_list1:\n",
    "  #   with open(output_file_name, \"a\") as f:print(\"'\",k,\"',\", sep='', file = f)\n",
    "  # with open(output_file_name, \"a\") as f:print(\"]\", file = f)\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if generate_feature_importance == 1:\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('Generating SHAP explanation')\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('')\n",
    "\n",
    "  with open(output_file_name, \"a\") as f:print('RF FEATURE IMPORTANCE',file = f)\n",
    "\n",
    "      #START TIMER MODEL\n",
    "  start = time.time()\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('Generating explainer')\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('')\n",
    "  test = X_test_01\n",
    "  train = X_train_01\n",
    "  # ## Summary Bar Plot Global\n",
    "  start_index = 0\n",
    "  end_index = 250\n",
    "  # test.pop('Label')\n",
    "  # test.pop('is_train')\n",
    "  # print(label2)\n",
    "\n",
    "\n",
    "  explainer = shap.TreeExplainer(rf)\n",
    "\n",
    "  shap_values = explainer.shap_values(test[start_index:end_index])\n",
    "  shap_obj = explainer(test[start_index:end_index])\n",
    "  shap.summary_plot(shap_values = shap_values,\n",
    "                    features = test[start_index:end_index],\n",
    "                  show=False)\n",
    "  # plt.clf()\n",
    "\n",
    "  # if feature_selection_bit == 1 # On\n",
    "  # pick_prob = 0 # set equal one to choose the dataset with probabilities, set to 0 to choose one with the classes.\n",
    "  if pick_prob == 1:\n",
    "    plt.savefig('RF_SHAP_NSL_prob_01.png')\n",
    "  elif pick_prob == 0:\n",
    "    plt.savefig('RF_SHAP_NSL_class_01.png')\n",
    "\n",
    "  else: None\n",
    "  plt.clf()\n",
    "\n",
    "\n",
    "  vals= np.abs(shap_values).mean(1)\n",
    "  feature_importance = pd.DataFrame(list(zip(train.columns, sum(vals))), columns=['col_name','feature_importance_vals'])\n",
    "  feature_importance.sort_values(by=['feature_importance_vals'], ascending=False,inplace=True)\n",
    "  feature_importance.head()\n",
    "  print(feature_importance.to_string())\n",
    "  with open(output_file_name, \"a\") as f:print('Feature Importance: ',feature_importance.to_string(),file = f)\n",
    "\n",
    "\n",
    "\n",
    "  end = time.time()\n",
    "  with open(output_file_name, \"a\") as f:print('ELAPSE TIME LIME GLOBAL: ',(end - start)/60, 'min',file = f)\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  # feature_importance_vals = 'feature_importance_vals'  # Replace with the name of the column you want to extract\n",
    "  feature_val = feature_importance['feature_importance_vals'].tolist()\n",
    "\n",
    "  # col_name = 'col_name'  # Replace with the name of the column you want to extract\n",
    "  feature_name = feature_importance['col_name'].tolist()\n",
    "\n",
    "\n",
    "  # for item1, item2 in zip(feature_name, feature_val):\n",
    "  #     print(item1, item2)\n",
    "\n",
    "\n",
    "  # Use zip to combine the two lists, sort based on list1, and then unzip them\n",
    "  zipped_lists = list(zip(feature_name, feature_val))\n",
    "  zipped_lists.sort(key=lambda x: x[1],reverse=True)\n",
    "\n",
    "  # Convert the sorted result back into separate lists\n",
    "  sorted_list1, sorted_list2 = [list(x) for x in zip(*zipped_lists)]\n",
    "\n",
    "  # for k in sorted_list1:\n",
    "  #   with open(output_file_name, \"a\") as f: print(\"df.pop('\",k,\"')\", sep='', file = f)\n",
    "\n",
    "  # with open(output_file_name, \"a\") as f:print(\"Trial_ =[\", file = f)\n",
    "  # for k in sorted_list1:\n",
    "  #   with open(output_file_name, \"a\") as f:print(\"'\",k,\"',\", sep='', file = f)\n",
    "  # with open(output_file_name, \"a\") as f:print(\"]\", file = f)\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6.15 ('HITL': conda)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.15"
  },
  "vscode": {
   "interpreter": {
    "hash": "c4fae9402861b04134511d9dc79f354d2f3b6fe67518e49507b0a2b7cbc7bed8"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
